#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import pywikibot, re, sys, argparse, time
import traceback
import unicodedata

import blib
from blib import getparam, rmparam, tname, pname, msg, errandmsg, site
from collections import OrderedDict

verbose = True

skip_lemma_pages = []

def check_re_sub(pagemsg, action, refrom, reto, text, numsub=1, flags=0):
  newtext = re.sub(refrom, reto, text, numsub, flags)
  if newtext == text:
    pagemsg("WARNING: When %s, no substitution occurred" % action)
  return newtext

# Make sure there is one trailing newline
def ensure_one_trailing_nl(text):
  return re.sub(r"\n*$", r"\n\n", text)

# Make sure there are two trailing newlines
def ensure_two_trailing_nl(text):
  return re.sub(r"\n*$", r"\n\n", text)

# Compare two values but first normalize them to composed form.
# This is important when comparing translits because translit taken directly
# from wikicode will be composed, whereas translit generated by expanding
# the {{xlit|yi|...}} template will be decomposed.
def compare_normalized(x, y):
  return unicodedata.normalize("NFC", x) == unicodedata.normalize("NFC", y)

# Return a tuple (RU, TR) with TR set to a blank string if it's redundant.
# If TR is already blank on entry, it is just returned.
def check_for_redundant_translit(yi, tr, pagemsg, expand_text):
  if not tr:
    return yi, tr
  autotr = expand_text("{{xlit|yi|%s}}" % yi)
  if not autotr:
    pagemsg("WARNING: Error generating translit for %s" % yi)
    return yi, tr
  if compare_normalized(autotr, tr):
    pagemsg("Removing redundant translit %s from Yiddish %s" % (tr, yi))
    return yi, ""
  pagemsg("Keeping non-redundant translit %s != auto %s for Yiddish %s" % (
    tr, autotr, yi))
  return yi, tr

# Return True if LEMMA (in the form RUSSIAN or RUSSIAN/TRANSLIT) matches the
# specified Cyrillic term RU, with possible manual transliteration TR
# (may be empty). Issue a warning if Cyrillic matches but not translit.
# FIXME: If either the lemma specifies manual translit or TR is given,
# we should consider transliterating the other one in case of redundant
# manual translit.
def lemma_matches(lemma, yi, tr, pagemsg, expand_text):
  if "//" in lemma:
    lemru, lemtr = re.split("//", lemma, 1)
  else:
    lemru, lemtr = lemma, ""
  if yi == lemru:
    # If one of the two has manual translit but the other doesn't, generate
    # translit for the one without it in case of redundant manual translit
    if tr and not lemtr:
      lemtr = expand_text("{{xlit|yi|%s}}" % lemru)
      if not lemtr:
        pagemsg("WARNING: Error generating translit for %s" % lemru)
        return False
    elif lemtr and not tr:
      tr = expand_text("{{xlit|yi|%s}}" % yi)
      if not tr:
        pagemsg("WARNING: Error generating translit for %s" % yi)
        return False
    trmatches = not tr and not lemtr or compare_normalized(tr, lemtr)
    if not trmatches:
      pagemsg("WARNING: Value %s matches lemma %s of yi-(proper )noun+, but translit %s doesn't match %s" % (
        yi, lemru, tr, lemtr))
    else:
      return True
  return False

pages_already_erased = set()

# Create or insert a section describing a given inflection of a given lemma.
# INFLECTIONS is the list of tuples of (INFL, INFLTR), i.e. accented
# inflectional form (e.g. the plural, feminine, verbal noun, participle,
# etc.) and associated manual transliteration (or None); LEMMA is the
# accented lemma (e.g. the singular, masculine or dictionary form of a
# verb); and LEMMATR is the associated manual transliterations (if any).
# POS is the part of speech of the word (capitalized, e.g. "Noun"). Only
# save the changed page if SAVE is true. INDEX is the numeric index of
# the lemma page, for ID purposes and to aid restarting. INFLTYPE is e.g.
# "adj form nom_m", and is used in messages; both POS and INFLTYPE are
# used in special-case code that is appropriate to only certain inflectional
# types. LEMMATYPE is e.g. "infinitive" or "masculine singular" and is
# used in messages.
#
# HEADTEMP is the headword template for the inflected-word entry (e.g.
# "head|yi|verb form"; we special-case "head|" headword templates).
# HEADTEMP_PARAM is a parameter or parameters to add to the created
# HEADTEMP template, and should be either empty or of the form
# "|foo=bar" (or e.g. "|foo=bar|baz=bat" for more than one parameter).
#
# DEFTEMP is the definitional template that points to the base form (e.g.
# "inflection of" or "yi-participle of"). DEFTEMP_PARAM is a parameter
# or parameters to add to the created DEFTEMP template, similar to
# HEADTEMP_PARAM; or it should be a list of inflection codes (e.g.
# ['2', 's', 'pres', 'ind']). DEFTEMP_NEEDS_LANG indicates whether the
# definition template specified by DEFTEMP needs to have a 'lang'/'1'
# parameter with value 'yi'. DEFTEMP_ALLOWS_MULTIPLE_TAG_SETS indicates
# whether multiple tag sets can be inserted into the definitional template
# (True for {{inflection of}}, currently false for {{yi-participle of}}).
#
# If ENTRYTEXT is given, this is the text to use for the entry, starting
# directly after the "==Etymology==" line, which is assumed to be necessary.
# If not given, this text is synthesized from the other parameters.
#
# IS_LEMMA_TEMPLATE is a function that is passed one argument, a template,
# and should indicate if it's a lemma template (e.g. 'yi-adj' for adjectives).
# This is used to issue warnings in case of non-lemma forms where there's
# a corresponding lemma (NOTE, this situation could be legitimate for nouns).
#
# LEMMAS_TO_OVERWRITE is a list of lemma pages the forms of which to overwrite
# the inflection codes of when an existing definition template (e.g.
# "inflection of") is found with matching lemma. Entries are without accents.
#
# LEMMAS_TO_NOT_OVERWRITE is a list of lemma pages, which should in general
# be the entire set of lemmas. Any non-lemma form that would overwrite the
# Yiddish section will not do so if the form is one of these pages.
# Entries are without accents.
#
# ALLOW_STRESS_MISMATCH_IN_DEFN is used when dealing with stress variants to
# allow for stress mismatch when inserting a new subsection next to an
# existing one, instead of creating a new etymology section.
def create_inflection_entry(program_args, save, index, inflections, lemma,
    lemmatr, pos, infltype, lemmatype, headtemp, headtemp_param, deftemp,
    deftemp_param, deftemp_allows_multiple_tag_sets=True,
    deftemp_needs_lang=True, entrytext=None, is_lemma_template=None,
    lemmas_to_overwrite=[], lemmas_to_not_overwrite=[]):

  # Remove any links that may esp. appear in the lemma, esp. if multiword.
  lemma = blib.remove_links(lemma)
  inflections = [(blib.remove_links(infl), infltr) for infl, infltr in inflections]

  joined_infls = ",".join(infl for infl, infltr in inflections)
  # Make this a function because it's needed in pagemsg(), but we may change
  # INFLTR down below (code is below because it needs pagemsg() to run).
  def joined_infls_with_tr():
    return ",".join("%s (%s)" % (infl, infltr) if infltr else "%s" % infl for infl, infltr in inflections)

  # Fetch pagename, create pagemsg() fn to output msg with page name included
  pagenames = set(infl for infl, infltr in inflections)
  # If multiple inflections, they should have the same pagename minus accents
  assert len(pagenames) == 1
  pagename = list(pagenames)[0]

  def pagemsg(text, simple=False, msgfun=msg):
    if simple:
      msgfun("Page %s %s: %s" % (index, pagename, text))
    else:
      msgfun("Page %s %s: %s: %s %s, %s %s%s" % (index, pagename, text, infltype,
        joined_infls_with_tr(), lemmatype, lemma, " (%s)" % lemmatr if lemmatr else ""))
  def pagemsg_if(doit, text, simple=False):
    if doit:
      pagemsg(text, simple=simple)
  def errandpagemsg(txt):
    pagemsg(txt, fun=errandmsg)
  def expand_text(tempcall):
    return blib.expand_text(tempcall, pagename, pagemsg, verbose)

  for (skip_form, skip_lemma) in skip_form_pages:
    if skip_form == pagename and skip_lemma == lemma:
      pagemsg("WARNING: Skipping form because in skip_form_pages for lemma %s"
          % skip_lemma)
      return

  # Remove any redundant manual translit
  lemma, lemmatr = check_for_redundant_translit(lemma, lemmatr, pagemsg, expand_text)
  inflections = [check_for_redundant_translit(infl, infltr, pagemsg, expand_text) for infl, infltr in inflections]

  is_participle = "_part" in infltype
  is_adj_form = "adjective form" in infltype
  is_noun_form = "noun form" in infltype
  is_verb_form = "verb form" in infltype
  is_short_adj_form = "adjective form short" in infltype
  is_noun_or_adj = "noun" in infltype or "adjective" in infltype
  is_noun_adj_plural = is_noun_or_adj and ("_p" in infltype or "_mp" in infltype)
  generic_infltype = ("participle" if is_participle else
      re.sub(" form.*", " form", infltype) if " form" in infltype else infltype)

  deftemp_uses_inflection_of = deftemp == "inflection of"
  headtemp_is_head = headtemp.startswith("head|")

  for infl, infltr in inflections:
    if infl == "-":
      pagemsg("Not creating %s entry - for %s %s%s" % (
        infltype, lemmatype, lemma, " (%s)" % lemmatr if lemmatr else ""))
      return

  # Prepare to create page
  pagemsg("Creating entry")
  page = pywikibot.Page(site, pagename)

  # Check whether parameter PARAM of template T matches VALUE.
  def compare_param(t, param, value, valuetr, issue_warnings=True):
    paramval = blib.remove_links(getparam(t, param))
    # If checking the lemma param, substitute page name if missing.
    if not paramval and param in ["1", "2", "head"]:
      paramval = pagename
    matches = paramval == value
    if matches:
      if param in ["1", "2", "head"]:
        trparam = "tr"
      elif param.startswith("head"):
        trparam = re.sub("^head", "tr", param)
      else:
        assert not valuetr, "Translit cannot be specified with a non-head parameter"
        return True
      trparamval = getparam(t, trparam)
      if not valuetr and not trparamval:
        return True
      if valuetr == trparamval:
        return True
      pagemsg_if(issue_warnings, "WARNING: Value %s matches param %s=%s, but translit %s doesn't match param %s=%s: %s" % (
        value, param, paramval, valuetr, trparam, trparamval, str(t)))
      return False
    return False

  # True if the heads in the template match all the inflections in INFLECTIONS,
  # a list of (FORM, FORMTR) tuples. Warn if some but not all match, and
  # warn if all match but some heads are left over. Knows how to deal with
  # yi-noun+ and yi-proper noun+.
  def template_head_matches(t, inflections, purpose, fail_when_left_over_heads=False, issue_warnings=True):
    some_match = False
    all_match = True
    left_over_heads = False

    # Get list of head params
    headparams = set()
    headparams.add("head")
    i = 1
    while True:
      i += 1
      param = "head" + str(i)
      if not getparam(t, param):
        break
      headparams.add(param)

    # Check to see whether all inflections match, and remove head params
    # that have matched so we can check if any are left over
    for infl, infltr in inflections:
      for param in headparams:
        if compare_param(t, param, infl, infltr, issue_warnings=issue_warnings):
          some_match = True
          headparams.remove(param)
          break
      else:
        all_match = False
    left_over_heads = headparams

    if some_match and not all_match:
      pagemsg_if(issue_warnings, "WARNING: Some but not all inflections %s match template when %s: %s" %
          (joined_infls_with_tr(), purpose, str(t)))
    elif all_match and left_over_heads:
      if fail_when_left_over_heads:
        pagemsg_if(issue_warnings, "WARNING: All inflections %s match template, but extra heads in template when %s, treating as a non-match: %s" %
            (joined_infls_with_tr(), purpose, str(t)))
        return False
      else:
        pagemsg_if(issue_warnings, "WARNING: All inflections %s match template, but extra heads in template when %s: %s" %
            (joined_infls_with_tr(), purpose, str(t)))
    return all_match

  # Prepare parts of new entry to insert
  if entrytext:
    entrytextl4 = re.sub("^==(.*?)==$", r"===\1===", entrytext, 0, re.M)
    newsection = "==Yiddish==\n\n===Etymology===\n" + entrytext
  else:
    # Synthesize new entry. Some of the parts here besides 'entrytext',
    # 'entrytextl4' and 'newsection' are used down below when creating
    # verb parts and participles; these parts don't exist when 'entrytext'
    # was passed in, but that isn't a problem because it isn't passed in
    # when creating verb parts or participles.

    # 1. Get the head=/1= and head2=,head3= etc. headword params.
    headparams = []
    headno = 0
    no_param1 = False
    if len(inflections) == 1 and inflections[0][0] == pagename and not inflections[0][1]:
      # Don't add head=/1= params if there's only one inflection that's the
      # same as the pagename and there's no translit.
      no_param1 = True
      pass
    else:
      for infl, infltr in inflections:
        headno += 1
        if headno == 1:
          headparams.append("|%s%s%s" % ("head=" if headtemp_is_head else "",
            infl, "|tr=%s" % infltr if infltr else ""))
        else:
          headparams.append("|head%s=%s%s" % (headno, infl,
            "|tr%s=%s" % (headno, infltr) if infltr else ""))

    # 3. Synthesize headword template.
    new_headword_template = "{{%s%s%s}}" % (headtemp, "".join(headparams),
        headtemp_param)

    # 4. Synthesize definition template.
    new_defn_template = "{{%s%s|%s%s%s}}" % (
      deftemp, "|yi" if deftemp_needs_lang else "",
      lemma, "|tr=%s" % lemmatr if lemmatr else "",
      deftemp_param if isinstance(deftemp_param, str) else "||" + "|".join(deftemp_param))

    # 5. Synthesize declension template if needed.
    newdecl = ""
    newdecll4 = ""

    # 6. Synthesize part of speech body and section text as a whole.
    newposbody = """%s

# %s
""" % (new_headword_template, new_defn_template)
    newpos = "===%s===\n" % pos + newposbody + newdecl
    newposl4 = "====%s====\n" % pos + newposbody + newdecll4
    entrytext = "\n" + newpos
    entrytextl4 = "\n" + newposl4
    newsection = "==Yiddish==\n" + entrytext

  comment = None
  notes = []

  # If invalid title, don't do anything.
  existing_text = blib.safe_page_text(page, errandpagemsg, bad_value_ret=None)
  if existing_text is None:
    return

  if not blib.safe_page_exists(page, errandpagemsg):
    # Page doesn't exist. Create it.
    pagemsg("Creating page")
    comment = "Create page for Yiddish %s %s of %s, pos=%s" % (
        infltype, joined_infls, lemma, pos)
    page.text = newsection
    if verbose:
      pagemsg("New text is [[%s]]" % page.text)
  else: # Page does exist
    pagetext = existing_text

    # Split off interwiki links at end
    m = re.match(r"^(.*?\n)(\n*(\[\[[a-z0-9_\-]+:[^\]]+\]\]\n*)*)$",
        pagetext, re.S)
    if m:
      pagebody = m.group(1)
      pagetail = m.group(2)
    else:
      pagebody = pagetext
      pagetail = ""

    # Split into sections
    splitsections = re.split("(^==[^=\n]+==\n)", pagebody, 0, re.M)
    # Extract off pagehead and recombine section headers with following text
    pagehead = splitsections[0]
    sections = []
    for i in range(1, len(splitsections)):
      if (i % 2) == 1:
        sections.append("")
      sections[-1] += splitsections[i]

    found_plurale_tantum_lemma = False

    # Go through each section in turn, looking for existing Yiddish section
    for i in range(len(sections)):
      m = re.match("^==([^=\n]+)==$", sections[i], re.M)
      if not m:
        pagemsg("Can't find language name in text: [[%s]]" % (sections[i]))
      elif m.group(1) == "Yiddish":
        # Extract off trailing separator
        mm = re.match(r"^(.*?\n)(\n*--+\n*)$", sections[i], re.S)
        if mm:
          # Note that this changes the number of sections, which is seemingly
          # a problem because the for-loop above calculates the end point
          # at the beginning of the loop, but is not actually a problem
          # because we always break after processing the Yiddish section.
          sections[i:i+1] = [mm.group(1), mm.group(2)]

        # Split off categories at end
        mm = re.match(r"^(.*?\n)(\n*(\[\[Category:[^\]]+\]\]\n*)*)$",
            sections[i], re.S)
        if mm:
          # See comment above.
          sections[i:i+1] = [mm.group(1), mm.group(2)]

        if program_args.overwrite_page:
          if pagename in pages_already_erased:
            pagemsg("WARNING: Not overwriting page, already overwritten previously")
          elif "==Etymology 1==" in sections[i] and not program_args.overwrite_etymologies:
            errandpagemsg("WARNING: Found ==Etymology 1== in page text, not overwriting, skipping form")
            return
          elif "{{audio|" in sections[i]:
            errandpagemsg("WARNING: {{audio|...}} in page text, not overwriting, skipping form")
            return
          elif pagename in lemmas_to_not_overwrite:
            errandpagemsg("WARNING: Page in --lemmas-to-not-overwrite, not overwriting, skipping form")
            return
          else:
            parsed = blib.parse_text(sections[i])
            found_lemma = []
            for t in parsed.filter_templates():
              tnam = tname(t)
              if tnam in ["yi-noun", "yi-proper noun", "yi-adj", "yi-verb",
                  "yi-adv", "yi-phrase"] or (tnam == "head" and
                      getparam(t, "1") == "yi" and getparam(t, "2") in
                      ["circumfix", "conjunction", "determiner", "interfix",
                        "interjection", "letter", "numeral", "cardinal number",
                        "particle", "predicative", "prefix", "preposition",
                        "prepositional phrase", "pronoun"]):
                found_lemma.append(getparam(t, "2") if tnam == "head" else
                    tnam)
            if found_lemma:
              errandpagemsg("WARNING: Page appears to have a lemma on it, not overwriting, skipping form: lemmas = %s"
              % ",".join(found_lemma))
              return
            notes.append("overwrite section")
            pagemsg("WARNING: Overwriting entire Yiddish section")
            # Preserve {{also|...}}
            sections[i] = re.sub(r"^((\s*\{\{also\|.*?\}\}\s*)?).*$", r"\1",
                sections[i], 0, re.S)
            pages_already_erased.add(pagename)

        # When creating non-lemma forms, warn about matching lemma template
        if is_lemma_template:
          parsed = blib.parse_text(sections[i])
          for t in parsed.filter_templates():
            if is_lemma_template(t):
              if template_head_matches(t, inflections, "checking for lemma"):
                pagemsg("WARNING: Creating non-lemma form and found matching lemma template: %s" % str(t))
              if is_noun_form:
                tnam = tname(t)
                if tnam in ["yi-noun", "yi-proper noun"] and any([re.search(r"\bp\b", x) for x in blib.fetch_param_chain(t, "2", "g")]):
                  found_plurale_tantum_lemma = True

        if found_plurale_tantum_lemma and is_noun_form and is_noun_adj_plural:
          pagemsg("WARNING: Found plurale tantum lemma and creating plural noun form, might need to add lemmas to allow_in_same_etym_section")

        subsections = re.split("(^===+[^=\n]+===+\n)", sections[i], 0, re.M)

        # We will loop through the subsections up to 4 times, seeing if we
        # can find a match or a way of inserting a new definition. If so, we
        # break the loop.
        #
        # Pass 0 = check for existing headword and definition, without adding
        #          a new gender to the headword; break if so
        # Pass 1 = same, but this time allow a new gender to be added;
        #          break if existing headword and definition found
        # Pass 2 = if no existing definition, check for ability to insert
        #          new defn into existing subsection without adding a new
        #          gender; break if so
        # Pass 3 = if got this far, check for ability to insert new defn
        #          into existing subsection adding a new gender
        #
        # If we got through all four passes without breaking, we will
        # proceed below to insert a new subsection, and possibly a whole
        # new etymology section or Yiddish-language section.
        #
        # We do this to handle certain situations where there are multiple
        # subsections. For example, моль has three separate etymologically
        # unrelated meanings, with different declensions and genders. моль
        # meaning "clothes moth" is feminine animate, while моль meaning
        # "minor (music)" or "mole (chemistry)" is masculine inanimate.
        # моли is gen/dat/pre sg and nom pl or моль "clothes moth", and
        # nom/acc pl of the other two meanings. After the gen sg and nom pl
        # were created by the bot, it was hand-edited and split into two
        # etymology sections, one feminine animate with definitions for
        # gen sg and nom pl and the second masculine inanimate with a
        # definition for nom pl. When processing the nom pl of the masculine
        # inanimate meanings, if we don't have separate passes with and
        # without allowing for adding gender, we'll match the first etymology
        # section and add masculine inanimate gender to it, when there's
        # already a suitable entry in the second etymology section. Similarly,
        # when processing the acc pl of the masculine inanimate meanings,
        # without separate passes we would insert the defn into the first
        # etymology section and modify the gender, when it should go into
        # the second. In other circumstances, without separate passes to
        # first check for existing definitions we might add a definition for
        # a form into an earlier subsection when there's already such a
        # definition in a later subsection.

        for process_section_pass in range(4):
          need_outer_break = False

          # Go through each subsection in turn, looking for subsection
          # matching the POS with an appropriate headword template whose
          # head matches the inflected form
          for j in range(2, len(subsections), 2):
            if re.match("^===+%s===+\n" % pos, subsections[j - 1]):
              # Found a POS match
              parsed = blib.parse_text(subsections[j])

              # Split a list of tags into individual tag sets,
              # where the individual tag sets are separated by a ; tag.
              def split_inflection_tag_sets(tags):
                tag_sets = []
                cur_tag_set = []
                for tag in tags:
                  if tag == ";":
                    tag_sets.append(cur_tag_set)
                    cur_tag_set = []
                  else:
                    cur_tag_set.append(tag)
                tag_sets.append(cur_tag_set)
                return tag_sets

              # Join tag sets back into a list of tags.
              def join_inflection_tag_sets(tag_sets):
                tags = []
                for tag_set in tag_sets:
                  if tags:
                    tags.append(";")
                  tags.extend(tag_set)
                return tags

              # Split a tag set possibly containing multipart tags
              # into one or more tag sets not containing such tags.
              def split_multipart_tag_set(ts):
                for i, tag in enumerate(ts):
                  if "//" in tag:
                    single_tags = tag.split("//")
                    pre_tags = ts[0:i]
                    post_tags = ts[i+1:]
                    tag_sets = []
                    for single_tag in single_tags:
                      tag_sets.extend(split_multipart_tag_set(
                        pre_tags + [single_tag] + post_tags))
                    return tag_sets
                return [ts]

              # Check for a given tag (including in part of a multipart tag)
              # in the tags of any definitional template (with the correct
              # language) in the subsection.
              # FIXME, should this check the lemma?
              def check_for_given_inflection_tag(parsed, tag):
                for t in parsed.filter_templates():
                  if tname(t) != deftemp:
                    continue
                  lang_in_1 = deftemp_needs_lang and not t.has("lang")
                  lang_param = lang_in_1 and "1" or "lang"
                  if (not deftemp_needs_lang or
                        compare_param(t, lang_param, "yi", None,
                          issue_warnings=issue_warnings)):
                    for param in t.params:
                      pnam = pname(param)
                      pvalue = str(param.value)
                      if (pnam not in ["1", "2"] and not (lang_in_1 and pnam == "3")
                          and re.search("^[0-9]+$", pnam)):
                        # Individual components may be separated by //
                        # (first-level) or : (second-level).
                        split_values = re.split("//|:", pvalue)
                        if tag in split_values:
                          return True
                return False

              # Replace the form-code parameters of tag set TAG_SET_NO
              # in "inflection of" (or "yi-participle of") with those
              # in INFLS, putting the non-form-code parameters in the
              # right places. If TAG_SET_NO is -1, add to the end.
              # if TAG_SET_NO == "all", replace all tag sets.
              def check_fix_defn_params(t, tag_set_no, infls):
                # Following code mostly copied from fix_verb_form.py
                origt = str(t)
                # Fetch lemma and alt params, and non-numbered params.
                lang = getparam(t, "lang")
                lang_in_1 = deftemp_needs_lang and not lang
                if lang_in_1:
                  lang = getparam(t, "1")
                lemmaparam = getparam(t, "2" if lang_in_1 else "1")
                altparam = getparam(t, "3" if lang_in_1 else "2")
                tr = getparam(t, "tr")
                tags = []
                non_numbered_params = []
                for param in t.params:
                  pnam = pname(param)
                  pvalue = str(param.value)
                  if (pnam not in ["1", "2"] and not (lang_in_1 and pnam == "3")
                      and re.search("^[0-9]+$", pnam) and pvalue):
                    tags.append(pvalue)
                  if not re.search(r"^[0-9]+$", pnam) and pnam not in ["lang", "tr"]:
                    non_numbered_params.append((pnam, param.value))

                tag_sets = split_inflection_tag_sets(tags)
                if tag_set_no == -1:
                  tag_sets.append(infls)
                elif tag_set_no == "all":
                  tag_sets = [infls]
                else:
                  tag_sets[tag_set_no] = infls
                tags = join_inflection_tag_sets(tag_sets)

                # Erase all params.
                del t.params[:]
                # Put back lang, lemma param, alt param, tr, then the
                # replacements for the higher numbered params, then the
                # non-numbered params. Use new-style params (lang in 1)
                # even if params were old-style (lang in lang=).
                if lang:
                  t.add("1", lang)
                t.add("2" if lang else "1", lemmaparam)
                t.add("3" if lang else "2", altparam)
                if tr:
                  t.add("tr", tr)
                for paramno, param in enumerate(tags):
                  t.add(str(paramno+(4 if lang else 3)), param)
                for name, value in non_numbered_params:
                  t.add(name, value)
                newt = str(t)
                if origt != newt:
                  pagemsg("Replaced %s with %s" % (origt, newt))
                  if tag_set_no != -1:
                    # FIXME, unnecessary long-term dependency here,
                    # where we happen to know that the places where
                    # tag_set_no is called with a non-negative
                    # number or "all" are used for updating aspect
                    # codes.
                    notes.append("update form codes (pfv/impfv)")
                  subsections[j] = str(parsed)
                  sections[i] = ''.join(subsections)

              # True if the tag sets in template T (an "inflection of"
              # template) exactly match the inflections given in INFLS (in
              # any order), or if the former are a superset of the latter.
              # Return value is a tuple (MATCH, TAG_SET_NO) where TAG_SET_NO
              # is the index of the tag set in template T that matches (0 if
              # there's only one tag set in T, but there may be multiple tag
              # sets separated by a semicolon; or "all") and MATCH is
              # * True if any tag set in T either exactly matches the
              #   inflections in INFLS (where "exactly match" takes into
              #   account tag sets with multipart tags, which are effectively
              #   multiple tag sets jammed together) or is a superset of the
              #   inflections in INFLS (in which case a warning is issued);
              # * "update" if an exact match isn't found and the current
              #   lemma is in lemmas_to_overwrite, or if an exact match or
              #   superset isn't found but an exact match would be found
              #   if "pfv" or "impfv" were added to a tag set;
              # * else False.
              def compare_inflections(t, infls, issue_warnings=True):
                lang_in_1 = deftemp_needs_lang and not t.has("lang")
                inflset = set(infls)
                tags = []
                for param in t.params:
                  name = pname(param)
                  value = str(param.value)
                  if (name not in ["1", "2"] and not (lang_in_1 and name == "3")
                      and re.search("^[0-9]+$", name) and value):
                    tags.append(value)
                tag_sets = split_inflection_tag_sets(tags)
                split_tag_sets = [split_multipart_tag_set(tag_set) for tag_set in tag_sets]
                # See if there's an exact match.
                for tag_set_no, split_tag_set_group in enumerate(split_tag_sets):
                  for indiv_tag_set in split_tag_set_group:
                    if set(indiv_tag_set) == inflset:
                      return True, tag_set_no

                # Return "update" if lemma in lemmas_to_overwrite.
                if lemma in lemmas_to_overwrite:
                  return "update", "all"

                # See if there's a superset match.
                for tag_set_no, split_tag_set_group in enumerate(split_tag_sets):
                  for indiv_tag_set in split_tag_set_group:
                    if set(indiv_tag_set) > inflset:
                      pagemsg_if(issue_warnings, "WARNING: Found actual inflection %s in template %s whose codes are a superset of intended codes %s, accepting" % (
                        "|".join(indiv_tag_set), str(t), "|".join(infls)))
                      return True, tag_set_no
                # See if there's a subset match.
                for tag_set_no, split_tag_set_group in enumerate(split_tag_sets):
                  for indiv_tag_set in split_tag_set_group:
                    indiv_set = set(indiv_tag_set)
                    if indiv_set < inflset:
                      # Check to see if we match except for a missing
                      # perfective or imperfective aspect, which we will update.
                      if (indiv_set | {"pfv"}) == inflset or (indiv_set | {"impfv"}) == inflset:
                        if len(split_tag_set_group) == 1:
                          pagemsg_if(issue_warnings, "Need to update actual inflection %s in template %s with intended codes %s" % (
                            "|".join(indiv_tag_set), str(t), "|".join(infls)))
                          return "update", tag_set_no
                        else:
                          pagemsg_if(issue_warnings, "WARNING: Found actual inflection %s in template %s whose codes are a subset of intended codes %s and could update aspect except that multipart tags are present" % (
                            "|".join(indiv_tag_set), str(t), "|".join(infls)))
                return False, 0

              # Find the inflection headword template(s) (e.g.
              # 'head|yi|verb form').
              def template_name(t):
                if headtemp_is_head:
                  return "|".join([tname(t), getparam(t, "1"), getparam(t, "2")])
                else:
                  return tname(t)

              # When checking to see if entry (headword and definition) already
              # present, allow extra heads, so e.g. when checking for 2nd pl
              # fut ind спали́те and we find a headword with both спали́те and
              # спа́лите we won't skip it. But when checking for headword without
              # definition to insert a new definition under it, make sure no
              # left-over heads, otherwise we will insert 2nd pl imperative
              # спали́те under the entry with both спали́те and спа́лите, which is
              # incorrect. (спали́те is both fut ind and imper, but спа́лите is
              # only fut ind. Many verbs work this way. The two forms of the
              # fut ind are found under separate conjugation templates so we
              # won't get a single request with both of them.)
              #
              # Set ISSUE_WARNINGS appropriately so we only issue warnings
              # on the first pass, rather than duplicating up to four times
              # on each pass.
              issue_warnings = process_section_pass == 0
              infl_headword_templates_for_already_present_entry = [
                  t for t in parsed.filter_templates()
                  if template_name(t) == headtemp and
                  template_head_matches(t, inflections,
                    "checking for already-present entry",
                    issue_warnings=issue_warnings)]
              infl_headword_templates_for_inserting_in_same_section = [
                  t for t in parsed.filter_templates()
                  if template_name(t) == headtemp and
                  template_head_matches(t, inflections,
                    "checking for inserting defn in same section",
                    fail_when_left_over_heads=True,
                    issue_warnings=issue_warnings)]

              def check_for_closely_related_lemma(otherlemma, lemma,
                  issue_warnings=True):
                for lemma1, lemma2 in allow_defn_in_same_subsection:
                  if (lemma1 == otherlemma and lemma2 == lemma or
                      lemma2 == otherlemma and lemma1 == lemma):
                    pagemsg_if(issue_warnings, "Allowing lemma %s to share headword with lemma %s because in allow_defn_in_same_subsection" %
                        (lemma, otherlemma))
                    return True
                return False

              # Find the definitional (typically "inflection of") template(s).
              # We store a tuple of (TEMPLATE, NEEDS_UPDATE) where NEEDS_UDPATE
              # is true if we need to overwrite the form codes (this happens
              # when we want to add the verb aspect 'pfv' or 'impfv' to the
              # form codes).
              defn_templates_for_already_present_entry = []
              defn_templates_for_inserting_in_same_section = []
              defn_templates_for_inserting_in_same_template = []
              for t in parsed.filter_templates():
                if tname(t) != deftemp:
                  continue
                lang_in_1 = deftemp_needs_lang and not t.has("lang")
                lang_param = lang_in_1 and "1" or "lang"
                lemma_param = lang_in_1 and "2" or "1"
                if (compare_param(t, lemma_param, lemma, lemmatr,
                      issue_warnings=issue_warnings) and
                    (not deftemp_needs_lang or
                      compare_param(t, lang_param, "yi", None,
                        issue_warnings=issue_warnings))):
                  defn_templates_for_inserting_in_same_section.append(t)
                  defn_templates_for_inserting_in_same_template.append(t)
                  if isinstance(deftemp_param, str):
                    defn_templates_for_already_present_entry.append((t, False, 0))
                  else:
                    result, tag_set_no = compare_inflections(t, deftemp_param,
                        issue_warnings=issue_warnings)
                    if result == "update":
                      defn_templates_for_already_present_entry.append((t, True, tag_set_no))
                    elif result:
                      defn_templates_for_already_present_entry.append((t, False, 0))
                # Also see if the definition template matches a closely-related
                # lemma where we allow the two to share the same headword
                # (e.g. огонь and alternative form огнь)
                elif (check_for_closely_related_lemma(getparam(t, lemma_param), lemma,
                      issue_warnings=issue_warnings) and
                    (not deftemp_needs_lang or
                      compare_param(t, lang_param, "yi", None,
                        issue_warnings=issue_warnings))):
                  defn_templates_for_inserting_in_same_section.append(t)

              # Check for singular in any existing definition templates
              # (with the correct language) in the subsection.
              # FIXME, should this check the lemma?
              singular_in_existing_defn_templates = (
                check_for_given_inflection_tag(parsed, "s"))

              # Make sure there's exactly one headword template.
              if (len(infl_headword_templates_for_already_present_entry) > 1
                  or len(infl_headword_templates_for_inserting_in_same_section) > 1):
                pagemsg("WARNING: Found multiple inflection headword templates for %s; taking no action"
                    % (infltype))
                need_outer_break = True
                break

              # We found both templates and their heads matched; inflection
              # entry is already present.
              if (infl_headword_templates_for_already_present_entry and
                  defn_templates_for_already_present_entry and
                  process_section_pass in [0, 1]):
                if process_section_pass == 0:
                  pagemsg("Exists and has Yiddish section and found %s already in it"
                      % (infltype))
                # Maybe override the current form code parameters in the
                # definition template(s) with the supplied ones (i.e. those
                # derived from the declension/conjugation template on the
                # lemma page).
                for t, needs_update, tag_set_no in defn_templates_for_already_present_entry:
                  if needs_update:
                    check_fix_defn_params(t, tag_set_no, deftemp_param)
                inserted = insert_part_decl_if_needed()
                if inserted:
                  comment = "Insert declension in existing entry: %s %s, %s %s" % (
                    infltype, joined_infls, lemmatype, lemma)
                else:
                  # "Do nothing", but set a comment, in case we made a
                  # template change like changing gender (NOTE: in addition
                  # to the comment, there are notes, which will reflect
                  # any minor changes like changing gender, so if we've
                  # taken a more major action like inserting a declension
                  # above, we don't need to put "Updating params" in the
                  # comment.)
                  comment = "Update params of existing entry: %s %s, %s %s" % (
                      infltype, joined_infls, lemmatype, lemma)
                need_outer_break = True
                break

              # At this point, didn't find either headword or definitional
              # template, or both. If we found headword template and another
              # definition template for the same lemma, insert new definition
              # in same section.
              elif (infl_headword_templates_for_inserting_in_same_section and
                  defn_templates_for_inserting_in_same_section and
                  process_section_pass in [2, 3]):
                # If multiple tag sets can be inserted into a single
                # definitional template, try to do that.
                if (deftemp_allows_multiple_tag_sets and
                    len(defn_templates_for_inserting_in_same_template) > 0 and
                    # FIXME, when is deftemp_param a string?
                    not isinstance(deftemp_param, str)):
                  check_fix_defn_params(defn_templates_for_inserting_in_same_template[-1],
                    -1, deftemp_param)
                  pagemsg("Insert new tag set into existing {{%s}}" % (
                      deftemp))
                  # FIXME, this might not occur in these circumstances
                  inserted = insert_part_decl_if_needed()
                  comment = "%s new tag set into existing {{%s}}: %s %s, %s %s" % (
                      "Insert declension in existing entry and insert" if inserted else "Insert",
                      deftemp, infltype, joined_infls, lemmatype, lemma)
                else:
                  # If there's already a defn line present, insert after
                  # any such defn lines. Else, insert at beginning.
                  if re.search(r"^# \{\{%s\|" % deftemp, subsections[j], re.M):
                    if not subsections[j].endswith("\n"):
                      subsections[j] += "\n"
                    subsections[j] = check_re_sub(pagemsg, "inserting definition into existing section",
                        r"(^(# \{\{%s\|.*\n)+)" % deftemp,
                        r"\1# %s\n" % new_defn_template, subsections[j],
                        1, re.M)
                  else:
                    subsections[j] = check_re_sub(pagemsg, "inserting definition into existing section",
                        r"^#", "# %s\n#" % new_defn_template,
                        subsections[j], 1, re.M)
                  sections[i] = ''.join(subsections)
                  pagemsg("Insert new defn with {{%s}} at beginning after any existing such defns" % (
                      deftemp))
                  inserted = insert_part_decl_if_needed()
                  comment = "%s new defn with {{%s}} at beginning after any existing such defns: %s %s, %s %s" % (
                      "Insert declension in existing entry and insert" if inserted else "Insert",
                      deftemp, infltype, joined_infls, lemmatype, lemma)
                need_outer_break = True
                break

          if need_outer_break:
            break

        # else of for loop over passes 0-3, i.e. no break out of loop
        else:
          # At this point we couldn't find an existing subsection with
          # matching POS and appropriate headword template whose head matches
          # the inflected form.

          def insert_new_text_before_section(insert_at, secbefore_desc,
              matching):
            pagemsg("Found section to insert %s before: [[%s]]" % (
                generic_infltype, subsections[insert_at + 1]))

            # Determine indent level
            m = re.match("^(==+)", subsections[insert_at])
            indentlevel = len(m.group(1))

            secmsg = "%s section for same %s" % (secbefore_desc, matching)
            pagemsg("Inserting before %s" % secmsg)
            comment = "Insert entry for %s %s of %s before %s" % (
              infltype, joined_infls, lemma, secmsg)
            if insert_at > 0:
              subsections[insert_at - 1] = ensure_two_trailing_nl(
                  subsections[insert_at - 1])
            if indentlevel == 3:
              subsections[insert_at:insert_at] = [newpos + "\n"]
            else:
              assert(indentlevel == 4)
              subsections[insert_at:insert_at] = [newposl4 + "\n"]
            sections[i] = ''.join(subsections)
            return comment

          def insert_new_text_after_section(insert_at, secafter_desc,
              matching):
            pagemsg("Found section to insert %s after: [[%s]]" % (
                generic_infltype, subsections[insert_at - 1]))

            # Determine indent level and skip past sections at higher indent
            m = re.match("^(==+)", subsections[insert_at - 2])
            indentlevel = len(m.group(1))
            while insert_at < len(subsections):
              if (insert_at % 2) == 0:
                insert_at += 1
                continue
              m = re.match("^(==+)", subsections[insert_at])
              newindent = len(m.group(1))
              if newindent <= indentlevel:
                break
              pagemsg("Skipped past higher-indented subsection: [[%s]]" %
                  subsections[insert_at])
              insert_at += 1

            secmsg = "%s section for same %s" % (secafter_desc, matching)
            pagemsg("Inserting after %s" % secmsg)
            comment = "Insert entry for %s %s of %s after %s" % (
              infltype, joined_infls, lemma, secmsg)
            subsections[insert_at - 1] = ensure_two_trailing_nl(
                subsections[insert_at - 1])
            if indentlevel == 3:
              subsections[insert_at:insert_at] = [newpos + "\n"]
            else:
              assert(indentlevel == 4)
              subsections[insert_at:insert_at] = [newposl4 + "\n"]
            sections[i] = ''.join(subsections)
            return comment

          # If adjective form, try to find an existing participle form with
          # the same headword to insert after. If short adjective form, also
          # try to find an existing adverb or predicative with the same
          # headword to insert after. In all cases, insert after the last such
          # one.
          if is_adj_form:
            insert_at = None
            for j in range(2, len(subsections), 2):
              if re.match("^===+Participle===+", subsections[j - 1]):
                parsed = blib.parse_text(subsections[j])
                for t in parsed.filter_templates():
                  if (tname(t) == "head" and getparam(t, "1") == "yi" and
                      getparam(t, "2") == "participle form" and
                      template_head_matches(t, inflections, "checking for existing participle with headword matching adjective")):
                    insert_at = j + 1
            if insert_at:
              comment = insert_new_text_after_section(insert_at,
                  "adverb/predicative/participle form" if is_short_adj_form
                  else "participle form", "headword")
              break

          # Check whether lemma LEMMA of form to add and already-existing
          # form of OTHERLEMMA are pairs in allow_in_same_etym_section.
          # If FIRST_ONLY, require that OTHERLEMMA is the first one
          # (the plurale tantum); we use this when creating a form that
          # will be in the same etym section as a plurale tantum lemma,
          # so we don't by accident end up inserting in the same etym section
          # as a normal singular lemma.
          def check_for_matching_sg_pl_pair(otherlemma, lemma,
              first_only=False):
            for lemma1, lemma2 in allow_in_same_etym_section:
              if (lemma1 == otherlemma and lemma2 == lemma or
                  not first_only and lemma2 == otherlemma and lemma1 == lemma):
                pagemsg("Allowing new subsection for lemma %s in same etym section as lemma %s because in allow_in_same_etym_section list" %
                    (lemma, otherlemma))
                return True
            return False

          def matching_defn_templates(parsed, check_for_sg_pl_pairs=False):
            retval = []
            for t in parsed.filter_templates():
              if tname(t) != deftemp:
                continue
              lang_in_1 = deftemp_needs_lang and not t.has("lang")
              lang_param = lang_in_1 and "1" or "lang"
              lemma_param = lang_in_1 and "2" or "1"
              if (
                (compare_param(t, lemma_param, lemma, lemmatr) or
                  check_for_sg_pl_pairs and check_for_matching_sg_pl_pair(
                    getparam(t, lemma_param), lemma)) and
                (not deftemp_needs_lang or
                  compare_param(t, lang_param, "yi", None))
              ):
                retval.append(t)
            return retval

          # If adjective form, try to find noun form with same headword
          # (inflection) and definition (lemma) to insert before (this happens
          # with nouns that are substantivized adjectives). Insert before
          # first such one.
          if is_adj_form:
            insert_at = None
            for j in range(2, len(subsections), 2):
              if re.match("^===+Noun===+", subsections[j - 1]):
                parsed = blib.parse_text(subsections[j])
                defn_templates = matching_defn_templates(parsed)
                for t in parsed.filter_templates():
                  if (tname(t) == "head" and getparam(t, "1") == "yi" and
                      getparam(t, "2") == "noun form" and
                      template_head_matches(t, inflections, "checking for existing noun form with headword and defn matching adj form") and
                      defn_templates and
                      insert_at is None):
                    insert_at = j - 1

            if insert_at is not None:
              comment = insert_new_text_before_section(insert_at, "noun form",
                  "headword and definition")
              break

          # If noun form, try to find adjective form with same headword
          # (inflection) and definition (lemma) to insert after (this happens
          # with nouns that are substantivized adjectives). Insert after
          # last such one.
          if is_noun_form:
            insert_at = None
            for j in range(2, len(subsections), 2):
              if re.match("^===+Adjective===+", subsections[j - 1]):
                parsed = blib.parse_text(subsections[j])
                defn_templates = matching_defn_templates(parsed)
                for t in parsed.filter_templates():
                  if (tname(t) == "head" and getparam(t, "1") == "yi" and
                      getparam(t, "2") == "adjective form" and
                      template_head_matches(t, inflections, "checking for existing adj form with headword and defn matching noun form") and
                      defn_templates):
                    insert_at = j + 1

            if insert_at:
              comment = insert_new_text_after_section(insert_at, "adj form",
                  "headword and definition")
              break

          # Try to find plurale tantum noun lemma for paired lemma in
          # allow_in_same_etym_section list (e.g. creating gen_sg/nom_pl
          # бакенбарды of бакенбарда, on same page as plurale tantum
          # бакенбарды). Insert after the last such one.
          if is_noun_form:
            insert_at = None
            for j in range(2, len(subsections), 2):
              if re.match("^===+Noun===+", subsections[j - 1]):
                parsed = blib.parse_text(subsections[j])
                for t in parsed.filter_templates():
                  if tname(t) in ["yi-noun", "yi-proper noun"]:
                    otherlemmas = blib.fetch_param_chain(t, "1", "head")
                    for otherlemma in otherlemmas:
                      if check_for_matching_sg_pl_pair(otherlemma, lemma,
                          first_only=True):
                        insert_at = j + 1

            if insert_at:
              comment = insert_new_text_after_section(insert_at,
                  "plurale tantum noun", "lemma pair in allow_in_same_etym_section")
              break

          # Now try to find an existing section corresponding to the same
          # lemma. This happens e.g. with verb forms, such as смо́трите
          # 2nd plural pres ind vs. смотри́те 2nd plural imperative, or
          # with nouns of e.g. accent patterns c and d, in the gen sg vs.
          # nom pl of masculine nouns.
          #
          # Insert after the last such section.

          insert_at = None
          for j in range(2, len(subsections), 2):
            if re.match("^===+%s===+\n" % pos, subsections[j - 1]):
              parsed = blib.parse_text(subsections[j])
              defn_templates = matching_defn_templates(parsed,
                  check_for_sg_pl_pairs=is_noun_form)
              if defn_templates:
                insert_at = j + 1

          if insert_at:
            comment = insert_new_text_after_section(insert_at, generic_infltype,
                "definition")
            break

          # Check for another plural noun form if we're a plural noun form
          if is_noun_form and is_noun_adj_plural:
            found_plural_noun_form = False
            for j in range(2, len(subsections), 2):
              if re.match("^===+Noun===+", subsections[j - 1]):
                parsed = blib.parse_text(subsections[j])
                # Check for singular in any existing definition templates
                # (with the correct language) in the subsection.
                # FIXME, should this check the lemma?
                plural_in_existing_defn_templates = (
                  check_for_given_inflection_tag(parsed, "p"))
                # Now check for matching yi-noun form, where either there was
                # a plural in a defn template or there's a plural gender
                # (the latter is necessary because plurale tantum forms don't
                # have "p" in the defn template)
                for t in parsed.filter_templates():
                  if (tname(t) in ["yi-noun form"] and
                      template_head_matches(t, inflections, "checking for plural noun form") and (
                        plural_in_existing_defn_templates or
                        any([re.search(r"\bp\b", y) for y in blib.fetch_param_chain(t, "2", "g")]))):
                    found_plural_noun_form = True
            if found_plural_noun_form or found_plurale_tantum_lemma:
              pagemsg("WARNING: Creating new etymology for plural noun form and found existing plural noun form or noun lemma")

          pagemsg("Exists and has Yiddish section, appending to end of section")
          # [FIXME! Conceivably instead of inserting at end we should insert
          # next to any existing ===Noun=== (or corresponding POS, whatever
          # it is), in particular after the last one. However, this makes less
          # sense when we create separate etymologies, as we do. Conceivably
          # this would mean inserting after the last etymology section
          # containing an entry of the same part of speech.
          #
          # (Perhaps for now we should just skip creating entries if we find
          # an existing Yiddish entry?)] -- comment out of date
          if not sections[i]: # Erased section
            pagemsg("Blank section, creating it")
            comment = "Create Yiddish section for %s %s of %s, pos=%s" % (
              infltype, joined_infls, lemma, pos)
            sections[i] = newsection
          elif "\n===Etymology 1===\n" in sections[i]:
            j = 2
            while ("\n===Etymology %s===\n" % j) in sections[i]:
              j += 1
            pagemsg("Found multiple etymologies, adding new section \"Etymology %s\"" % (j))
            comment = "Append entry (Etymology %s) for %s %s of %s, pos=%s in existing Yiddish section" % (
              j, infltype, joined_infls, lemma, pos)
            sections[i] = ensure_two_trailing_nl(sections[i])

            sections[i] += "===Etymology %s===\n" % j + entrytextl4
          else:
            pagemsg("Wrapping existing text in \"Etymology 1\" and adding \"Etymology 2\"")
            comment = "Wrap existing Yiddish section in Etymology 1, append entry (Etymology 2) for %s %s of %s, pos=%s" % (
                infltype, joined_infls, lemma, pos)

            for j in range(2, len(subsections), 2):
              if re.match("^===+Etymology===+\n", subsections[j - 1]):
                pagemsg("Found Etymology section at position %s-%s" % (
                    j - 1, j))
                # Found etymology section; if there is a preceding section
                # such as Alternative forms, put the etymology section above
                # it.
                if j > 2:
                  pagemsg("Found Etymology section at position %s-%s, below other sections, moving up" % (
                    j - 1, j))
                  etymtext = subsections[j - 1:j + 1]
                  del subsections[j - 1:j + 1]
                  subsections[1:1] = etymtext
                  sections[i] = "".join(subsections)
                break

            # Wrap existing text in "Etymology 1" and increase the indent level
            # by one of all headers
            sections[i] = re.sub("^\n*==Yiddish==\n+", "", sections[i])
            wikilink_re = r"^(\{\{wikipedia\|.*?\}\})\n*"
            mmm = re.match(wikilink_re, sections[i])
            wikilink = (mmm.group(1) + "\n") if mmm else ""
            if mmm:
              sections[i] = re.sub(wikilink_re, "", sections[i])
            sections[i] = re.sub("^===Etymology===\n", "", sections[i])
            sections[i] = ("==Yiddish==\n" + wikilink + "\n===Etymology 1===\n" +
                ("\n" if sections[i].startswith("==") else "") +
                ensure_two_trailing_nl(re.sub("^==(.*?)==$", r"===\1===",
                  sections[i], 0, re.M)) +
                "===Etymology 2===\n" + entrytextl4)
        break
      elif m.group(1) > "Yiddish":
        pagemsg("Exists; inserting before %s section" % (m.group(1)))
        comment = "Create Yiddish section and entry for %s %s of %s, pos=%s; insert before %s section" % (
            infltype, joined_infls, lemma, pos, m.group(1))
        sections[i:i] = [newsection, "\n----\n\n"]
        break

    else: # else of for loop over sections, i.e. no break out of loop
      pagemsg("Exists; adding section to end")
      comment = "Create Yiddish section and entry for %s %s of %s, pos=%s; append at end" % (
          infltype, joined_infls, lemma, pos)

      if sections:
        sections[-1] = ensure_two_trailing_nl(sections[-1])
        sections += ["----\n\n", newsection]
      else:
        if not program_args.overwrite_page:
          notes.append("formerly empty")
        if pagehead.lower().startswith("#redirect"):
          pagemsg("WARNING: Page is redirect, overwriting")
          notes.append("overwrite redirect")
          pagehead = re.sub(r"#redirect *\[\[(.*?)\]\] *(<!--.*?--> *)*\n*",
              r"{{also|\1}}\n", pagehead, 0, re.I)
        elif not program_args.overwrite_page:
          pagemsg("WARNING: No language sections in current page")
        sections += [newsection]

    # End of loop over sections in existing page; rejoin sections
    newtext = pagehead + ''.join(sections) + pagetail

    if page.text != newtext:
      assert comment or notes

    # Eliminate newlines at the end of the text, because that is done
    # automatically when saving, and doing this avoids some unnecessary saves
    newtext = re.sub(r"\n*$", "", newtext)
    # Eliminate sequences of 3 or more newlines, which may come from
    # ensure_two_trailing_nl(). Add comment if none, in case of existing page
    # with extra newlines.
    newnewtext = re.sub(r"\n\n\n+", r"\n\n", newtext)
    if newnewtext != newtext and not comment and not notes:
      notes = ["eliminate sequences of 3 or more newlines"]
    newtext = newnewtext

    if page.text == newtext:
      pagemsg("No change in text")
    elif verbose:
      pagemsg("Replacing <%s> with <%s>" % (page.text, newtext),
          simple = True)
    else:
      pagemsg("Text has changed")
    page.text = newtext

  # Executed whether creating new page or modifying existing page.
  # Check for changed text and save if so.
  notestext = '; '.join(notes)
  if notestext:
    if comment:
      comment += " (%s)" % notestext
    else:
      comment = notestext
  if page.text != existing_text:
    if save:
      blib.safe_page_save(page, comment, errandpagemsg)
    else:
      pagemsg("Would save with comment = %s" % comment, simple=True)

# Parse a noun/verb/adv form spec (from the user), one or more forms separated
# by commas, possibly including aliases. INFL_DICT is a dictionary
# mapping possible form codes to a tuple specifying the corresponding set of
# inflection codes in {{inflection of|...}}, or a list of multiple such tuples
# (for cases where a single form code refers to multiple inflections, such
# as with adjectives, where the form code gen_m specifies not only the genitive
# masculine singular but also the genitive neuter singular and the animate
# accusative masculine singular. ALIASES is a dictionary mapping aliases to
# form codes. Returns a list of tuples (FORM, INFLSETS), where FORM is a form
# code and INFLSETS is the corresponding value entry in INFL_DICT (a tuple of
# inflection codes, or a list of such tuples).
def parse_form_spec(formspec, infl_dict, aliases):
  forms = []
  for form in re.split(",", formspec):
    if form in aliases:
      for f in aliases[form]:
        if f not in forms:
          forms.append(f)
    elif form in infl_dict:
      if form not in forms:
        forms.append(form)
    else:
      raise ValueError("Invalid value '%s'" % form)

  infls = []
  for form in forms:
    infls.append((form, infl_dict[form]))
  return infls

adj_form_inflection_list = [
  # used with all variants
  ["n", "acc//dat|m|s|;|def//postpositive|dat|n|s"],
  ["er", "nom|m|s|;|dat|f|s"],
  ["e", "def|nom//acc|n|s|;|nom//acc|f|s|;|all-case|p"],
  ["s", "postpositive|nom//acc|n|s"],
]

def adj_form_inflection_dict(infltemp, args):
  return dict(adj_form_inflection_list)

adj_form_aliases = {
    "all":[x for x, y in adj_form_inflection_list],
}

poss_adj_form_inflection_list = [
  ["n", "postpositive|acc//dat|m|s|;|postpositive|dat|n|s"],
  ["er", "postpositive|nom|m|s|;|postpositive|dat|f|s"],
  ["e", "postpositive|nom//acc|f|s|;|all-case|p"],
  ["s", "postpositive|nom//acc|n|s"],
]

def poss_adj_form_inflection_dict(infltemp, args):
  return dict(poss_adj_form_inflection_list)

poss_adj_form_aliases = {
    "all":[x for x, y in poss_adj_form_inflection_list],
}

def split_yi_tr(form):
  if "//" in form:
    yitr = re.split("//", form)
    assert len(yitr) == 2
    yi, tr = yitr
    return (yi, tr)
  else:
    return (form, None)

# Find inflection templates, skipping those under SKIP_POSES and issuing
# warnings for bad headers and bad level indentation, according to
# EXPECTED_HEADER and EXPECTED_POSES (see comment to create_forms()).
# Return a list of tuples of TEMPLATE.
def find_inflection_templates(text, expected_header, expected_poses, skip_poses,
    is_inflection_template, pagemsg, expand_text):
  templates = []

  sections = re.split("(^==[^=\n]+==\n)", text, 0, re.M)
  for i in range(2, len(sections), 2):
    if sections[i-1] == "==Yiddish==\n":
      subsections = re.split("(^===+[^=\n]+===+\n)", sections[i], 0, re.M)
      headers_at_level = {}
      last_levelno = 2
      for j in range(2, len(subsections), 2):
        m = re.search("^(=+)([^=\n]+)", subsections[j-1])
        levelno = len(m.group(1))
        header = m.group(2)
        headers_at_level[levelno] = header
        if levelno - last_levelno > 1:
          pagemsg("WARNING: Misformatted header level (jump by %s - %s = %s, in section %s)" % (
            levelno, last_levelno, levelno - last_levelno, subsections[j-1].replace("\n", "")))
        last_levelno = levelno
        parsed = blib.parse_text(subsections[j])
        for t in parsed.filter_templates():
          if is_inflection_template(t) and not getparam(t, "old").strip():
            if header != expected_header:
              pagemsg("WARNING: Expected inflection template under %s header but instead found under %s header" % (
                expected_header, header))
            pos_header = headers_at_level.get(levelno-1, None)
            if pos_header and pos_header not in expected_poses:
              pagemsg("WARNING: Inflection template under unexpected part of speech %s" %
                  pos_header)
            if pos_header not in skip_poses:
              templates.append(t)
            else:
              pagemsg("Skipping inflection template because under part of speech %s: %s" % (
                pos_header, str(t)))
  return templates

# Create required forms for all nouns/verbs/adjectives.
#
# LEMMAS_TO_PROCESS is a list of lemma pages to process. Entries are assumed
# to be without accents. If empty, process all lemmas of the appropriate part of speech.
#
# LEMMAS_TO_OVERWRITE is a list of lemma pages the forms of which to overwrite
# the inflection codes of when an existing definition template (e.g.
# "inflection of") is found with matching lemma. Entries are without accents.
#
# LEMMAS_TO_NOT_OVERWRITE is a list of lemma pages, which should in general
# be the entire set of lemmas. Any non-lemma form that would overwrite the
# Yiddish section will not do so if the form is one of these pages.
# Entries are without accents.
#
# SAVE is as in create_inflection_entry(). STARTFROM and UPTO, if not None,
# delimit the range of pages to process (inclusive on both ends).
#
# FORMSPEC specifies the form(s) to do, a comma-separated list of form codes,
# possibly including aliases (e.g. 'all'). GENERATE_INFLECTION_DICT is a
# function of two arguments (the inflection template and a dictionary of forms)
# that returns a dictionary mapping possible form codes to a tuple of the
# corresponding inflection codes in {{inflection of|...}}, or a list of such
# tuples; see 'parse_form_spec'. FORM_ALIASES is a dictionary mapping aliases
# to form codes.
#
# POS specifies the part of speech (lowercase, singular, e.g. "verb").
# HEADTEMP specifies the headword template name (e.g. "head|yi|verb form").
# DICFORM_CODES specifies the form code for the dictionary
# form (e.g. "infinitive", "nom_m") or a list of such codes to try (e.g.
# ["nom_sg", "nom_pl"]).
#
# NOTE: There is special-case code that depends on the part of speech.
#
# EXPECTED_HEADER specifies the header that the inflection template (e.g.
# 'yi-decl-adj' for adjectives, 'yi-conj' for verbs) should be under
# (Declension or Conjugation); a warning will be issued if it's wrong.
# EXPECTED_POSES is a list of the parts of speech that the inflection template
# should be under (e.g. ["Noun", "Proper noun"]); a warning will be issued if
# an unexpected part of speech is found. A warning is also issued if the
# level indentation is wrong. SKIP_POSES is a list of parts of speech to skip
# the inflections of (e.g. ["Participle", "Pronoun"] for adjectives).
# IS_INFLECTION_TEMPLATE is a function that is passed one argument, a template,
# and should indicate if it's an inflection template. GENERATE_FORMS
# is a function that's passed two arguments, an inflection template and
# an 'expand_text' function, and should return an expansion of the template
# into a string identifying the set of forms, of the form
# 'FORMCODE1=VALUE1|FORMCODE2=VALUE2|...'.
#
# IS_LEMMA_TEMPLATE is a function that is passed one argument, a template,
# and should indicate if it's a lemma template (e.g. 'yi-adj' for adjectives).
# This is used to issue warnings in case of non-lemma forms where there's
# a corresponding lemma (NOTE, this situation could be legitimate for nouns).
#
# SKIP_INFLECTIONS, if supplied, should be a function of three arguments, the
# form name, Yiddish and translit (which may be missing), and should return
# true if the particular form value in question is to be skipped. This is
# used e.g. to skip periphrastic future forms.
def create_forms(lemmas_to_process, lemmas_to_overwrite,
    lemmas_to_not_overwrite, program_args, save, startFrom, upTo, formspec,
    generate_inflection_dict, form_aliases, pos, headtemp, dicform_codes,
    expected_header, expected_poses, skip_poses, is_inflection_template,
    generate_forms, is_lemma_template, skip_inflections=None):

  if type(dicform_codes) is not list:
    dicform_codes = [dicform_codes]

  if lemmas_to_process:
    pages_to_process = ((index, pywikibot.Page(site, page)) for index, page in
        blib.iter_items(lemmas_to_process, startFrom, upTo))
  else:
    pages_to_process = blib.cat_articles("Yiddish %ss" % pos, startFrom, upTo)

  for index, page in pages_to_process:
    pagetitle = str(page.title())
    def pagemsg(txt):
      msg("Page %s %s: %s" % (index, pagetitle, txt))
    def errandpagemsg(txt):
      errandmsg("Page %s %s: %s" % (index, pagetitle, txt))
    def expand_text(tempcall):
      return blib.expand_text(tempcall, pagetitle, pagemsg, verbose)
    if pagetitle.startswith("-"):
      pagemsg("Skipping suffix entry")
      continue
    if pagetitle in skip_lemma_pages:
      pagemsg("WARNING: Skipping lemma because in skip_lemma_pages")
      continue

    # Find the inflection templates. Rather than just look for all inflection
    # templates, we may skip those under certain parts of speech, e.g.
    # participles for adjective forms. This is to avoid the issue with
    # преданный, which has one adjectival inflection as an adjective
    # and a different one as a participle.
    inflection_templates = find_inflection_templates(page.text, expected_header,
        expected_poses, skip_poses, is_inflection_template,
        pagemsg, expand_text)
    if len(inflection_templates) > 1 and pos == "adjective":
      pagemsg("WARNING: Multiple inflection templates for %s" % pagetitle)

    for infltemp in inflection_templates:
      result = generate_forms(infltemp, expand_text)
      if not result:
        pagemsg("WARNING: Error generating %s forms, skipping" % pos)
        continue
      args = blib.split_generate_args(result)
      for dicform_code in dicform_codes:
        if dicform_code in args:
          break
      else:
        pagemsg("WARNING: No dictionary form available among putative codes %s, skipping" %
            ",".join(dicform_codes))
        continue
      if dicform_code != dicform_codes[0]:
        pagemsg("create_forms: Using non-default dictionary form code %s" % dicform_code)
      if "," in args[dicform_code]:
        pagemsg("WARNING: Can't handle comma in dictionary form code %s, skipping" %
          args[dicform_code])
        continue
      dicforms = [args[dicform_code]]
      forms_desired = parse_form_spec(formspec, generate_inflection_dict(infltemp, args),
        form_aliases)
      # Fetch dictionary forms, remove accents on monosyllables
      dicforms = [split_yi_tr(dicform) for dicform in dicforms]
      dicforms_args_sets = [(dicforms, args)]
      split_dicforms = dicforms
      split_args = args
      for dicformyi, dicformtr in split_dicforms:
        for formname, inflsets in forms_desired:
          # Skip the dictionary form; also skip forms that don't have
          # listed inflections (e.g. singulars with plural-only nouns,
          # animate/inanimate variants when a noun isn't bianimate):
          if formname != dicform_code and formname in split_args and split_args[formname]:
            # Group inflections by unaccented Yiddish, so we process
            # multiple accent variants together
            formvals_by_pagename = OrderedDict()
            formvals = re.split(",", split_args[formname])
            if len(formvals) > 1:
              pagemsg("create_forms: Found multiple form values for %s=%s, dictionary form %s%s" %
                  (formname, split_args[formname], dicformyi, dicformtr and " (%s)" % dicformtr or ""))
            for formval in formvals:
              formvalyi, formvaltr = split_yi_tr(formval)
              formval_no_accents = formvalyi
              if skip_inflections and skip_inflections(formname, formvalyi, formvaltr):
                pagemsg("create_forms: Skipping %s=%s%s" % (formname, formvalyi,
                  formvaltr and " (%s)" % formvaltr or ""))
              elif formval_no_accents in formvals_by_pagename:
                formvals_by_pagename[formval_no_accents].append((formvalyi, formvaltr))
              else:
                formvals_by_pagename[formval_no_accents] = [(formvalyi, formvaltr)]
            # Process groups of inflections
            formvals_by_pagename_items = list(formvals_by_pagename.items())
            if len(formvals_by_pagename_items) > 1:
              pagemsg("create_forms: For form %s, found multiple page names %s" % (
                formname, ",".join("%s" % formval_no_accents for formval_no_accents, inflections in formvals_by_pagename_items)))
            for formval_no_accents, inflections in formvals_by_pagename_items:
              inflections_printed = ",".join("%s%s" %
                  (infl, " (%s)" % infltr if infltr else "")
                  for infl, infltr in inflections)

              if formval_no_accents == dicformyi:
                pagemsg("create_forms: Skipping form %s=%s because would go on lemma page" % (formname, inflections_printed))
              else:
                if len(inflections) > 1:
                  pagemsg("create_forms: For pagename %s, found multiple inflections %s" % (
                    formval_no_accents, inflections_printed))

                if type(inflsets) is not list:
                  inflsets = [inflsets]
                for inflset in inflsets:
                  # For plurale tantum nouns, don't include "plural" in
                  # inflection codes.
                  if pos == "noun" and dicform_code == "nom_pl":
                    inflset = tuple(x for x in inflset if x != "p")
                  # For numerals, don't include "singular" or "plural"
                  # in inflection codes when the numerals don't vary
                  # according to number.
                  if pos == "numeral" and numeral_is_tantum(infltemp, dicform_code):
                    inflset = tuple(x for x in inflset if x not in ["s", "p"])
                  header_pos = pos.capitalize()
                  deftemp = "inflection of"
                  deftemp_needs_lang = True
                  deftemp_allows_multiple_tag_sets = True
                  our_headtemp = headtemp
                  create_inflection_entry(program_args, save, index,
                    inflections, dicformyi, dicformtr, header_pos,
                    "%s form %s" % (pos, formname), "dictionary form",
                    our_headtemp, "", deftemp, inflset,
                    is_lemma_template=is_lemma_template,
                    lemmas_to_overwrite=lemmas_to_overwrite,
                    lemmas_to_not_overwrite=lemmas_to_not_overwrite,
                    deftemp_needs_lang=deftemp_needs_lang,
                    deftemp_allows_multiple_tag_sets=deftemp_allows_multiple_tag_sets)

def generate_adj_forms(t, expand_text):
  if tname(t) == "yi-decl":
    return expand_text(re.sub(r"^\{\{yi-decl", "{{yi-generate-adj-forms", str(t)))
  else:
    assert tname(t) == "yi-decl-poss"
    return expand_text(re.sub(r"^\{\{yi-decl-poss\s*\|", r"{{yi-generate-adj-forms|decltype=possessive|",
      str(t)))

def create_adj_forms(save, startFrom, upTo, formspec, lemmas_to_process,
    lemmas_to_overwrite, lemmas_to_not_overwrite, program_args):
  create_forms(lemmas_to_process, lemmas_to_overwrite,
      lemmas_to_not_overwrite, program_args, save, startFrom, upTo, formspec,
      adj_form_inflection_dict, adj_form_aliases,
      "adjective", "head|yi|adjective form", "pred",
      "Declension", ["Adjective", "Participle", "Ordinal numeral"],
      ["Participle", "Pronoun", "Determiner"],
      lambda t:tname(t) in ["yi-decl", "yi-decl-poss"],
      generate_adj_forms,
      lambda t:tname(t) == "yi-adj",
      )

pa = blib.create_argparser("Create Yiddish inflection entries")
pa.add_argument("--adj-form",
    help="""Do specified adjective-form inflections, a comma-separated list.
Possible elements are pred, er, n, e, s or all (all forms). The predicate
form will not be created even if specified, because it is the same as the
dictionary/lemma form. Also, non-existent forms for particular adjectives
will not be created.""")
pa.add_argument("--lemmafile",
    help="""File containing lemmas to process.""")
pa.add_argument("--lemmas",
    help="""Comma-separated list of lemmas to process.""")
pa.add_argument("--overwrite-lemmas",
    help="""File containing list of lemmas where the current inflections are
considered to have errors in them (e.g. due to the conjugation template having
incorrect aspect) and thus should be overwritten. Entries are without
accents.""")
pa.add_argument("--lemmas-to-not-overwrite",
    help="""File containing list of lemma pages, which should in general
be the entire set of lemmas. Any non-lemma form that would overwrite the
Yiddish section (--overwrite-page) will not do so if the form is one of
these pages. Entries are without accents.""")
pa.add_argument("--overwrite-page", action="store_true",
    help="""If specified, overwrite the entire existing page of inflections.
Won't do this if it finds "Etymology N", unless --overwrite-etymologies is
given. WARNING: Be careful!""")
pa.add_argument("--overwrite-etymologies", action="store_true",
    help="""If specified and --overwrite-page, overwrite the entire existing
page of inflections even if "Etymology N". WARNING: Be careful!""")

params = pa.parse_args()
startFrom, upTo = blib.parse_start_end(params.start, params.end)

if params.lemmafile:
  lemmas_to_process = list(blib.yield_items_from_file(params.lemmafile))
elif params.lemmas:
  lemmas_to_process = blib.split_utf8_arg(params.lemmas)
else:
  lemmas_to_process = []
if params.overwrite_lemmas:
  lemmas_to_overwrite = list(blib.yield_items_from_file(params.overwrite_lemmas))
else:
  lemmas_to_overwrite = []
if params.lemmas_to_not_overwrite:
  lemmas_to_not_overwrite = list(blib.yield_items_from_file(params.lemmas_to_not_overwrite))
else:
  lemmas_to_not_overwrite = []
if params.adj_form:
  function_to_call = create_adj_forms
  function_to_call(params.save, startFrom, upTo, params.adj_form, lemmas_to_process, lemmas_to_overwrite, lemmas_to_not_overwrite, params)

blib.elapsed_time()
