#! /usr/bin/python3 # Last edited on 2026-09-09 03:42:23 by stolfi # The command line arguments are a the name {ivt_name} of an input file, # a text unit type {utype} ("ch" or "ec"), and the name {kword_en} of a # bipattern {bipat}, like "USES" or "BLOOD", that is recognized by # {get_bencao_starps_bipattern} in {standard_bipatterns.py}. # # Reads the file "res/{ivt_name}.ivt" The file should by # an IVTFF-like format, in the format "<{LOC}> {TEXT}". Finds all # occurrences of a given bipattern {bipat} in the {TEXT} of # each line, assuming that ist nature is that described by {utype}. # # Writes to "res/{ivt_name}-{kword_en}.wpp" a line for each such occurrence. # Each line has the format "{LOC} {PSIZE} {WPOS} {KPENA} {KWORD}", where # # {LOC} is the locus ID of the input line (parag, page). # # {PSIZE} is the length of the {TEXT} in that line. # # {WPOS} is the the position of the occurence of the word # namely the length of the part of the cleaned input {TEXT} that # precedes that occurrence. # # {KPENA} is a penalty associated with that particular variant # of the keyword pattern {bipattern}. # # {KWORD} is the actual substring of the cleaned {TEXT} that # matched by the bipattern {bipat}. # # The unit type {utype} specifies the nature of the {TEXT} and of the # units used when measuring line sizes and match positions. In # particuler, this script assumes that the {TEXT} has been cleaned-up # according to the {utype}. See {clean_up_raw_text} in # {size_position_funcs.py} for details. It also specifies which branch # of {bipat} should be considered. # # In any case the input file is assumed to be in Unicode UTF-8 encoding, # and so will be the output file. # # Each pattern in the appropriate branch of {bipat} is searched on the # {TEXT} as asubtring. In the "ec" case, the SPS branch of {bipat} # must match only the characters '[a-z?]'. In the "ch" case it must # match only simplified hanzi characters. In eiter case it must not # contain any punctuation or the special patterns '^', '$', and '\b'. # Raw parag sizes and positions are measured in EVA or hanzi characters. # # In any case, the {PSIZE} is the totalcount of units in the {TRXT}, # and {WPOS} is counts the count of units of {TEXT} that precede the matched # substring. import sys, os, re from sys import stderr as err from process_funcs import bash, basic_line_loop from error_funcs import arg_error, file_line_error, prog_error from chinese_funcs import read_chinese_char_set from note_077_funcs import compute_and_print_stats, name_for_tex_macro import size_position_funcs as spf import bipattern_match_funcs as bpf import standard_bipatterns as stdbip def main(ivt_name, utype, kword_en): # {ivt_name} Name of input ".ivt" file without extension or folder. # {utype} Unit for raw parag lengths and word positions: "ch" or "ec". # {kword_en} name of a bipattern, like "USES" or "BLOOD". assert utype == "ch" or utype == "ec", f"invalid {utype = !r}" assert re.fullmatch(r"[-A-Z0-9]+", kword_en) is not None, f"invalid {kword_en = !r}" bipat = stdbip.get_bencao_starps_bipattern(kword_en) in_file = f"res/{ivt_name}.ivt" rd = open(in_file, "r") rd.reconfigure(encoding='utf-8') out_file = f"res/{ivt_name}-{kword_en}.wpp" wr = open(out_file, "w") wr.reconfigure(encoding='utf-8') wr.write("# -*- coding: utf-8 -*-\n") pat_line, pat_unit, pat_sepa, clean_sepa = spf.get_parsing_patterns(utype) ch_trimmed = False # Assumed for ths program. del_qo = True # !!! Make parameter? !!! exp_irm = True # !!! Make parameter? !!! hanzi_per_unit = spf.default_hanzi_per_unit(utype, ch_trimmed, del_qo, exp_irm) tot_line = 0 # Count of data lines. tot_wocc = 0 # Total occurrences of {kword_en} found. tot_tlen = 0 # Total length of all texts. loc_list = [] # Locus IDs of the input lines, without [<>]. psize_list = [] # List of all unit counts of input lines. hits_list_list = [] # List of lists of triples {(wpos,kword,kpena)}. def process_input_line(nline, line): nonlocal tot_line, tot_wocc, tot_tlen nonlocal loc_list, psize_list, hits_list_list # # Parses a line {line} assuming it is line {nline} of the file. # The {line} is always a string (never {None}), but may be "" if the line # is empty. # # Ignores the line if it is a blank or #-comment. # # Otherwise the line must be a data line, matching {pat_line} # # Increments {tot_line} for each data line. # # Makes a list {hits_list} of triples {(wpos,kword,kpena)} for the # occurrences of the bipattern {bipat} counted as specified by # {utype}. # # For each data line, appends its data to {loc_list}, {psize_list?}, # and {hits_list_list}. # Should we debug the line? debug = False def data_error(msg): nonlocal in_file, nline, line file_line_error(in_file, nline, msg, line) assert False # ---------------------------------------------------------------------- assert line != None, "The {line} arg must not be {None}" # Ignore comments and blank lines: if re.match(r" *([#]|$)", line): return # Just in case, ignore IVTFF page headers: if re.match(r"", line): return tot_line += 1 m = re.match(pat_line, line) if m is None: # Invalid line format. data_error("invalid line format") # Parse the line into locus ID and text: assert m.lastindex == 2, f"bug {m.lastindex = }" loc = m.group(1) text = m.group(2) # Clean up the text, just in case: clean_text = spf.clean_up_raw_text(text, utype, data_error) clean_text = spf.normalize_text(clean_text, utype, del_qo, exp_irm, data_error) # Compute raw line size and raw occurrences of {bipat}: psize = len(clean_text) tot_tlen += psize hits_list = bpf.list_bipat_occurrences(clean_text, utype, bipat) tot_wocc += len(hits_list) # Store for processing at end: loc_list.append(loc) psize_list.append(psize) hits_list_list.append(hits_list) if debug: err.write(f"!~ {loc:<12s} {psize = } occs = {hits_list}\n") return # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: err.write(f"reading file '{in_file}' ...\n") nread = basic_line_loop(rd, process_input_line) rd.close() tot_line = len(loc_list) assert len(hits_list_list) == tot_line avg_wocc = tot_wocc/tot_line # Average occurrences per line. err.write(f"{nread:6d} lines read\n") err.write(f"{tot_line:6d} data lines found\n") err.write(f"{tot_wocc:5d} total text length {tot_tlen}\n") err.write(f"{tot_wocc:5d} total occurrences of {kword_en!r}\n") err.write(f"{avg_wocc:8.2f} avg occurrences/line\n") err.write(f"writing {out_file} with occurrences of {kword_en!r} ...\n") output_data(wr, loc_list, psize_list, hits_list_list) wr.close() write_TeX_parms_file \ ( ivt_name, hanzi_per_unit, kword_en, psize_list, hits_list_list, tot_line, tot_tlen, tot_wocc ) return # ---------------------------------------------------------------------- def output_data(wr, loc_list, psize_list, hits_list_list): debug = False for loc, psize, hits_list in zip(loc_list, psize_list, hits_list_list): if len(hits_list) > 0: for kwpos, kword, kpena in hits_list: wr.write("%-12s %6.2f %6.2f %7.5f %s\n" % (loc, psize, kwpos, kpena, kword)) wr.write("\n") wr.flush() return # ---------------------------------------------------------------------- def write_TeX_parms_file \ ( ivt_name, hanzi_per_unit, kword_en, psize_list, hits_list_list, tot_line, tot_tlen, tot_wocc ): # Writes the file "res/{ivt_name}-{kword_en}-woc-parms.tex" with # parameter defintions for LaTeX. # List ana analyze the COUNT of occurrences per input file line: noc_list = [ len(hits_list) for hits_list in hits_list_list ] noc_num, noc_tot, noc_min, noc_sin, noc_max, noc_sax, noc_avg, noc_dev = \ compute_and_print_stats("occurrences of the bipattern per line", noc_list) assert noc_num == tot_line assert noc_tot == tot_wocc err.write("\n") # List and analyze all POSITIONS of the occurrences: wpo_list = [ hit[0] for hits_list in hits_list_list for hit in hits_list ] wpo_num, wpo_tot, wpo_min, wpo_sin, wpo_max, wpo_sax, wpo_avg, wpo_dev = \ compute_and_print_stats("Positions of word per line", wpo_list) assert wpo_num == tot_wocc # Count lines with and without the word: tot_line_with_bipat = 0; for sub in hits_list_list: if len(sub) > 0: tot_line_with_bipat += 1 tot_line_sans_bipat = tot_line - tot_line_with_bipat err.write(f"lines with keyword = {tot_line_with_bipat} without = {tot_line_sans_bipat}\n") # Prefix for TeX macro names: txpref = name_for_tex_macro(f"{ivt_name}-{kword_en}-word-pos") tex_file = f"res/{ivt_name}-{kword_en}-kwpos-parms.tex" tex_wr = open(tex_file, "w") # These should match the defs from other TeX parms files: tex_wr.write(f"\\def\\{txpref}NumLines{{{tot_line}}}\n") tex_wr.write(f"\\def\\{txpref}TotChars{{{tot_tlen}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}LinesWith{{{tot_line_with_bipat}}}\n") tex_wr.write(f"\\def\\{txpref}LinesSans{{{tot_line_sans_bipat}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}HanziPerUnit{{{hanzi_per_unit:.3f}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}Min{{{wpo_min:.2f}}}\n") tex_wr.write(f"\\def\\{txpref}Max{{{wpo_max:.2f}}}\n") tex_wr.write(f"\\def\\{txpref}Avg{{{wpo_avg:.2f}}}\n") tex_wr.write(f"\\def\\{txpref}Dev{{{wpo_dev:.2f}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}TotCt{{{tot_wocc}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}PerLineMinCt{{{noc_min}}}\n") tex_wr.write(f"\\def\\{txpref}PerLineMaxCt{{{noc_max}}}\n") tex_wr.write(f"\\def\\{txpref}PerLineAvgCt{{{noc_avg:.2f}}}\n") tex_wr.write(f"\\def\\{txpref}PerLineDevCt{{{noc_dev:.2f}}}\n") tex_wr.write(f"\n") tex_wr.write(f"\\def\\{txpref}PerLineSecMinCt{{{noc_sin}}}\n") tex_wr.write(f"\\def\\{txpref}PerLineSecMaxCt{{{noc_sax}}}\n") tex_wr.write(f"\n") tex_wr.close() return # ---------------------------------------------------------------------- def test_stuff(): arg_error("no tests yet\n") return # ---------------------------------------------------------------------- if sys.argv[1] == "test": test_stuff() else: narg = len(sys.argv) iarg = 1 ivt_name = sys.argv[iarg]; iarg += 1 utype = sys.argv[iarg]; iarg += 1 kword_en = sys.argv[iarg]; iarg += 1 assert iarg == narg, f"spurious arguments = {sys.argv[iarg:]!r}" main(ivt_name, utype, kword_en)