#! /usr/bin/python3

import sys, re
import html_gen as h
from process_funcs import bash
import html_report_funcs as hr

last_edit = "Last edited on 2026-03-03 01:24:18 by stolfi"

def main():

  global last_edit
  
  title = "[096] Glossary for analysys of Herbal A vs. B"
  st = h.new_doc(title, "#eeffdd")
  
  h.section(st, 2, "Transcription files")
  
  h.parags(st, """All statistics on this paper are based on a <a href="join25e1.ivt">digital transcription of the VMS</a> in the EVA transcription alphabet which is partly Rene Zandbergen's recent IVT file and partly a new transcription of my own.  In this specific file, all #-comments and inline comments have been removed, the braces '{...}' indicating ligatures have been omitted, all "weirdo" character codes '&NNN;' have been mapped to '<tt>?</tt>', and all EVA codes have been mapped to loowecase.  Paragraphs are indicated with a '<tt><%></tt>' prefix on the head lines and '<tt><$></tt>' suffix on the tail lines.  Rene's IVTFF 'location codes' are not used; instead, on each parag or title line, one of the characters '<tt>«</tt>', '<tt>=</tt>', or '<tt>»</tt>' is used as prefix (resp. suffix) to indicate indentation relative to the left (resp. right) rail line of the enclosing text block, respectively. See the sample below:""")
  
  h.append_preformatted(st, """
      <f1r.7;U> <%>»?.odar.?.shol.cphoy.oydar.sh??s.cfhoaiin.shodar,y=
      <f1r.8;U> =yshey.shody.o,kchoy.otchol.chocthy.oschy.dain.chor.kos=
      <f1r.9;U> =daiin.shos.cfhol.shody«<$>
      <f1r.10;U> »dain.or.teody=
      <f1r.11;U> <%>»?.y,dain.cphesaiin.ol,s.cphey.y,tain.shoshy.cphodal,es=
      <f1r.12;U> =ok,sho.kshoy.o,tairin.otchl.okan.shodain.sckhhy.daiin=
""", ind = 4, centered = False)
  
  h.parags(st, """Note that lines 7-9 are a complete parag; lines 7 and 8 span the full width of the text block,
  the tail line 9 is left-justified but ends before the right rail, and line 10 is a right-justified title.
  
  For this note we apply a few additions cleanup actions on this file:""")
  h.begin_enum(st, "ul")
  h.enum_item(st, "The rare characters [<tt>bjuvxy</tt>] were mapped to '?' (\"invalid\").")
  h.enum_item(st, "The '<tt>g</tt>' in word-final position was replaced by '<tt>m</tt>'")
  h.enum_item(st, "The combination '<tt>ih</tt>' was replaced by '<tt>ch</tt>'.")
  h.enum_item(st, "The '<tt>i</tt>' at the start of platform gallows like '<tt>ith</tt>' was replaced by '<tt>c</tt>'.")
  h.enum_item(st, "The combination '<tt>hh</tt>' in platform gallows like '<tt>cthh</tt>' was replaced by '<tt>he</tt>'.")
  h.end_enum(st, "ul")
  
  h.parags(st, """After this cleanup, the transcribed Voynichese text consists entirely of the EVA letters [<tt>ac-fhik-ty</tt>], the invalid glyph marker '?', and the separators '-' (line break), '.' (definite word space), and ',' (uncertain word space).""")
  
  h.section(st, 2, "Sections and text types")
  
  h.parags(st, """For this note, the transcription file above was split into separate files "{sec}-{txtype}.ivt" in <a href="st_files">this forler</a> for each <i>section</i> {sec} ("hea", "heb","zod", etc.) and each <i>text type</i> {txtype}.  The assignment of VMS lines to sections is defined by <a href="ivt_loc_to_sec.tbl">this table</a>.  In this note we consider only the two parts of the Herbal section traditionally assigned to language A ("hea", 1203 lines) and language B ("heb", 384 lines).
  
  The text type of each line of the transcription file is defined by <a href="ivt_loc_to_type.tbl">another table</a>.  The possible types are "parags" (text in paragraph-like blocks), "trings" (text rings in diagrams), "radios" (radial lines in diagrams), "titles" (isolated multiword lines not part of parags), "glyphs" (isolated single cracaters not inside other text), and "labels" (short texts associated with figure details).  In this note we use only "parags" type text.""")
  
  h.section(st, 2, "Words, tokens, and WFDs")
  
  h.parags(st, """In this this article, we define a <i>word</i> (aka "word type") as a distinct string of EVA codes in [<tt>a-z?</tt>].  A <i>token</i> is an occurrence of a word in a specific position in a specific line of the transcription, delimied by word space characters [<tt>-,.</tt>].  Thus, for example, the text line <tt>chod.daiin,chody.daiin-chod</tt> contains three words (<tt>chod</tt>, <tt>chody</tt>, and <tt>daiin</tt>) but five tokens.
  
  The <i>count</i> {Ct(W)} of a word {W} in a section is the number of times it occurs as a token in that section.  Its <i>frequency</i> of <i>freq</i> {Fr(W)} is its count {Ct(W)} divided by the total numeber of tokens in the secton.  The function {Fr} is the <i>word frequency distribution</i> (WFD) of that section section.   The set of all words with nonzero frequency are the <i>lexicon</i> of the section.
  
  For this analysis we actually let {Ct(W)} be a <i>fractional count</i>, which is the expected count of <i>W</i> if each comma is replaced by a period or omitted, independently and with equal probability.  Thus, for example, in the line <tt>ar,ol-ody,dy,chy.daiin</tt> the word <tt>daiin</tt> has count 1, the words <tt>ar</tt>, <tt>ol</tt>, <tt>arol</tt>, <tt>ody></tt>, and <tt>chy</tt> have count of 0.5 each, and <tt>odydy</tt>, <tt>dy</tt>, <tt>dychy</tt>, and <tt>odydychy</tt> have counts 0.25 each.  The total number of tokens in that text is then defined as the sum of these fractional counts, namely 4.5.  Which is the number of dots and dashes in the text, plus half the number of commas, plus 1.""")

  h.output_doc(st, sys.stdout, 99, last_edit)
  return 0
  # ----------------------------------------------------------------------

main()