#! /usr/bin/gawk -f
# Last edited on 2026-09-29 17:13:32 by stolfi

# Caller must define with "-i" the function {factor_text}
# and with "-v" the variable {utype} as "ps" or "wp".

BEGIN { 
  if (utype == "") { arg_error("must define {utype}"); }
  ntoks = 0; # Total tokens.
  ngarb = 0; # Tokens with '?' on input.
  nbugs = 0; # Tokens with no '?' but violating model.
}

/^ *([#]|$)/ { print; next; }

/^<[bf][0-9A-Za-z.;]+>/ { 
  loc = $1; text = substr($0, length(loc)+1);
  printf "%s", loc;
  if (utype == "ps")
    { # Isolated pinyin syllables with diacritic or numeric tones:
      if (substr(loc,2,1) != "b")
        { data_error("bad locator"); }
    }
  else if (utype == "wp")
    { # EVA tokens, ignore commas:
      if (substr(loc,2,1) != "f")
        { data_error("bad locator"); }
      
      gsub(/[-.]/, " ", text);
      gsub(/[,]/, "", text);
      
      # printf "%s\n", text > "/dev/stderr"

      # Split some probable joined words:
      gsub(/\<aiinal\>/,        "aiin al", text)      
      gsub(/\<aiind\>/,         "aiin d", text)       
      gsub(/\<aiiraly\>/,       "aiir aly", text)     
      gsub(/\<airal\>/,         "air al", text)       
      gsub(/\<airam\>/,         "air am", text)       
      gsub(/\<airod\>/,         "air od", text)       
      gsub(/\<airody\>/,        "air ody", text)      
      gsub(/\<airol\>/,         "air ol", text)       
      gsub(/\<airoy\>/,         "air oy", text)       
      gsub(/\<chedykar\>/,      "chedy kar", text)    
      gsub(/\<cheolchey\>/,     "cheol chey", text)   
      gsub(/\<cheolkaiin\>/,    "cheol kaiin", text)  
      gsub(/\<cheolkain\>/,     "cheol kain", text)   
      gsub(/\<cheolkedy\>/,     "cheol kedy", text)   
      gsub(/\<chodchy\>/,       "chod chy", text)     
      gsub(/\<cholchedy\>/,     "chol chedy", text)   
      gsub(/\<cholcheol\>/,     "chol cheol", text)   
      gsub(/\<cholchey\>/,      "chol chey", text)    
      gsub(/\<cholkaiin\>/,     "chol kaiin", text)   
      gsub(/\<cholkain\>/,      "chol kain", text)    
      gsub(/\<cholkar\>/,       "chol kar", text)     
      gsub(/\<cholkedy\>/,      "chol kedy", text)    
      gsub(/\<cholkeedy\>/,     "chol keedy", text)   
      gsub(/\<cholkeey\>/,      "chol keey", text)    
      gsub(/\<cholkor\>/,       "chol kor", text)     
      gsub(/\<cholky\>/,        "chol ky", text)      
      gsub(/\<chorchy\>/,       "chor chy", text)     
      gsub(/\<daiindy\>/,       "daiin dy", text)     
      gsub(/\<daiinody\>/,      "daiin ody", text)    
      gsub(/\<daiiral\>/,       "daiir al", text)     
      gsub(/\<dairair\>/,       "dair air", text)     
      gsub(/\<dairal\>/,        "dair al", text)      
      gsub(/\<dairam\>/,        "dair am", text)      
      gsub(/\<dairar\>/,        "dair ar", text)      
      gsub(/\<dairin\>/,        "dair in", text)      
      gsub(/\<dairody\>/,       "dair ody", text)     
      gsub(/\<dairy\>/,         "dair y", text)       
      gsub(/\<kairam\>/,        "kair am", text)      
      gsub(/\<kalchdy\>/,       "kal chdy", text)     
      gsub(/\<kalchedy\>/,      "kal chedy", text)    
      gsub(/\<kalkal\>/,        "kal kal", text)      
      gsub(/\<kchokchy\>/,      "kcho kchy", text)    
      gsub(/\<kolchedy\>/,      "kol chedy", text)    
      gsub(/\<korchy\>/,        "kor chy", text)      
      gsub(/\<ogam\>/,          "og am", text)        
      gsub(/\<okairy\>/,        "okair y", text)      
      gsub(/\<okalchedy\>/,     "okal chedy", text)   
      gsub(/\<okalchy'\>/,      "okal chy", text)     
      gsub(/\<okalchy\>/,       "okal chy", text)     
      gsub(/\<okalody\>/,       "okal ody", text)     
      gsub(/\<okolchy\>/,       "okol chy", text)     
      gsub(/\<opalkaiin\>/,     "opal kaiin", text)   
      gsub(/\<otairor\>/,       "otair or", text)     
      gsub(/\<otalkedy\>/,      "otal kedy", text)    
      gsub(/\<otolchey\>/,      "otol chey", text)    
      gsub(/\<otorchy\>/,       "otor chy", text)     
      gsub(/\<otytam\>/,        "oty tam", text)      

      gsub(/\<polchdy\>/,       "pol chdy", text)     
      gsub(/\<polched\>/,       "pol ched", text)     
      gsub(/\<polchedy\>/,      "pol chedy", text)    
      gsub(/\<polshy\>/,        "pol shy", text)      

      gsub(/\<qokaiinos\>/,     "qokaiin os", text)   
      gsub(/\<qokalchdy\>/,     "qokal chdy", text)   
      gsub(/\<qokalchedy\>/,    "qokal chedy", text)  
      gsub(/\<qokarary\>/,      "qokar ary", text)    
      gsub(/\<qokolchedy\>/,    "qokol chedy", text)  
      gsub(/\<qotolchy\>/,      "qotol chy", text)    
      gsub(/\<sairom\>/,        "sair om", text)      
      gsub(/\<sairy\>/,         "sair y", text)       
      gsub(/\<sheolkain\>/,     "sheol kain", text)   
      gsub(/\<sholkal\>/,       "shol kal", text)     
      gsub(/\<sholkeedy\>/,     "shol keedy", text)   
      gsub(/\<tolchedy\>/,      "tol chedy", text)    
      gsub(/\<tolchor\>/,       "tol chor", text)     
      gsub(/\<tolshey\>/,       "tol shey", text)     
      gsub(/\<tolshy\>/,        "tol shy", text)      
      gsub(/\<torchey\>/,       "tor chey", text)     
      gsub(/\<torshor\>/,       "tor shor", text)     
      gsub(/\<ycheeolkaiin\>/,  "ycheeol kaiin", text)
    }
  n = split(text, fld) 
  buggy = 0
  for (i = 1; i <= n; i++) 
    { x = fld[i]
      y = factor_text(x);
      if (match(x, /[?]/)) 
        { ngarb++; }
      else
        { if (match(y, /[?]/)) { nbugs++; buggy = 1; }
        }
      printf " %s", y;
      ntoks ++;
    }
  printf " \n";
  if (buggy) { printf "%s\n", loc > "/dev/stderr"; }
  next
}

// { 
  file_line_error(FILENAME, FNR, "invalid line format", $0)
}

END{ 
  printf "%d unreadable or weirdo tokens in %d", ngarb, ntoks > "/dev/stderr"
  printf " (%.3f%%)\n", 100*ngarb/ntoks  > "/dev/stderr"
  printf "%d tokens with bad form in %d", nbugs, ntoks-ngarb > "/dev/stderr"
  printf " (%.3f%%)\n", 100*nbugs/(ntoks-ngarb)  > "/dev/stderr"
}