#!/usr/bin/python3
# encoding: utf-8
#
# wmllint -- check WML for conformance to the most recent dialect
#
# By Eric S. Raymond April 2007.
#
#                              PURPOSE
#
# wmllint is a tool for porting and debugging WML (Wesnoth Markup Language).
# It can do much of the work of upgrading old content to a new version of
# Battle for Wesnoth, automatically.
#
# While the script is at it, it checks for various incorrect and dodgy WML
# constructs, including:
#   * unbalanced tags
#   * strings that need a translation mark and do not have them
#   * strings that have a translation mark and should not
#   * translatable strings containing macro references
#   * filter references by id= not matched by an actual unit
#   * consistency between recruit= and recruitment_pattern= instances
#   * unknown unit types in recruitment lists
#   * double space after punctuation in translatable strings.
#   * unknown races or movement types in units
#   * unknown base units
#
#                              BASIC PROCEDURE
#
# Takes any number of directories as arguments.  Each directory is converted.
# If no directories are specified, acts on the current directory.
#
# The recommended procedure is this:
# 1. Run it with --dryrun first to see what it will do.
# 2. If the messages look good, run without --dryrun; the old content
#    will be left in backup files with a -bak extension.
# 3. Eyeball the changes with the --diff option.
# 4. Use wmlscope, with a directory list including the Wesnoth mainline WML
#    as first argument, to check that you have no unresolved references.
# 5. Test the conversion.
# 6. Use either --clean to remove the -bak files or --revert to
#    undo the conversion.
#
# Compatibility with pre-1.4 versions has been removed; to port very old
# content, first run wmllint-1.4 to convert to "1.4", before running this
# one.
#
#
#                              MAGIC COMMENTS
#
# wmllint: ignore                             - suppresses most wmllint checks on the line this comment is on.
# wmllint: noconvert                          - disables and suppresses output for changing the line the comment is on.
# wmllint: markcheck on|off                   - disables checking for missing or extra translation marks in the block between off and on.
# wmllint: no-icon                            - disables adding the name as the description (which also determines the icon) to attacks missing the description attribute.
#                                               used in the current and subsequent files that are processed.
# wmllint: usage of <unit> is <class>         - explicitly declares the recruitment type of the unit (see the usage attribute of [unit_type]) in case wmllint doesn't find it.
#                                               <unit> must be inside ascii double quotes.
# wmllint: usagetype <class>                  - tell wmllint about custom recruitment types. <class> can be a comma separated list.
# wmllint: validate-on|off                    - suppresses most wmllint checks in the block between off and on.
# wmllint: no translatables                   - suppresses output for files without a textdomain on the first line.
# wmllint: unbalanced-on|off                  - suppresses output when there are more opening tags than closing tags or vice versa in the block between off and on.
# wmllint: deathcheck on|off                  - suppresses output when a unit speaks in their death event in the block between off and on.
# wmllint: no ellipsecheck                    - suppresses output when finding unit ellipses that the engine adds automatically or custom unit ellipses that need to be updated.
# wmllint: conditional tag <tag_name>         - adds <tag_name> to the list of known conditional WML tags. used when defining custom conditional tags via lua.
#                                               <tag_name> is a comma delimited list. do not put square brackets around the tags listed in <tag_name>.
# wmllint: translatable <attribute>           - this adds <attribute> to the list of translatable attributes for the file.
#                                               prevents the warning that an attribute should not have a translation mark.
#
#                              DEVELOPER INFORMATION
#
# All conversion logic for lifting WML and maps from older versions of the
# markup to newer ones should live here.  This includes resource path changes
# and renames, also map format conversions.
#
# Note: Lift logic for pre-1.4 versions has been removed; if you need it,
# use wmllint-1.4 to lift before running this one.  I did this for a policy
# reason; I wanted to kill off the --oldversion switch.  It will *not*
# be restored; in future, changes to WML syntax *must* be forward
# compatible in such a way that tags from old versions can be
# unambiguously recognized (this will save everybody heartburn).  As a
# virtuous side effect, this featurectomy cuts wmllint's code
# complexity by over 50%, improves performance by about 33%, and
# banishes some annoying behaviors related to the 1.2 map-conversion
# code.
#

import sys, os, re, argparse, difflib, time, tomllib
from wesnoth.wmltools3 import *
from wesnoth.wmliterator3 import *
from wesnoth import version

# Changes meant to be done on maps and .cfg lines.
mapchanges = (
    ("^Voha", "^Voa"),
    ("^Voh",  "^Vo"),
    ("^Vhms", "^Vhha"),
    ("^Vhm",  "^Vhh"),
    ("^Vcha", "^Vca"),
    ("^Vch",  "^Vc"),
    ("^Vcm",  "^Vc"),
    ("Qv",  "Mv"),

    # new to 1.17.8
    ("Ior", "Iwo"),
    ("Icn", "Iwc"),
    ("Irs", "Isr"),
    ("Ias", "Isa"),
    ("Icr", "Isc"),
    )

# These terrains were changed from just base to base + overlay
# Automatic replacement is possible only
# when the old terrain doesn't have an overlay
# The terrain name is used to print warnings
mapchanges_with_overlay = (
    ("Xol", "Xos^Efs", "enlightened wall"),
    ("Ggf", "Gg^Efm", "grassland flowers"),
    )

# Base terrain aliasing changes.
aliaschanges = (
    # 1.11.8:
    ("Ch",   "Ct"),
    ("Ds",   "Dt"),
    ("Hh",   "Ht"),
    ("Mm",   "Mt"),
    ("Ss",   "St"),
    ("Uu",   "Ut"),
    ("Ww",   "Wst"),
    ("Wo",   "Wdt"),
    ("Wwr",  "Wrt"),
    ("^Uf",  "Uft"),
    # Vi -> Vit in 1.11.8, Vit -> Vt in 1.11.9.
    ("Vit",  "Vt"),
    # 1.11.9:
    ("Vi",   "Vt"),
    )

# Deprecated lines that ought to be removed globally. Suppressed by noconvert.
obsolete_lines = (
    "{MAGENTA_IS_THE_TEAM_COLOR}",
    "{SOUND:SLOW}",
    "{SOUND:POISON}",
    "{NO_INTERRUPT_NO_UNDO}",
    "{ENABLE_NIGHTBLADE}",
    )

# things that need to be updated but aren't handled by wmllint
warn_if_found = (
    ("{DRAKE_FLYING_ANIM}", "This is for the old style Drake animations. If you still need this for some reason, copy the macro definition into your add-on."),
    ("{STORY_PART_SPEECH}", "This no longer works properly, use the title attribute for the caption instead"),
    ("{EARLY_FINISH_BONUS_NOTE}", "This is now automatically generated and not required. Use [objectives][gold_carryover]bonus=yes instead."),
    ("{NO_EARLY_FINISH_BONUS_NOTE}", "This is now automatically generated and not required. Use [objectives][gold_carryover]bonus=no instead."),
    ("{NO_GOLD_CARRYOVER_NOTE}", "This is now automatically generated and not required. Use [objectives][gold_carryover]carryover_percentage=0 instead."),
    ("{NEW_GOLD_CARRYOVER_NOTE_100}", "This is now automatically generated and not required. Use [objectives][gold_carryover]carryover_percentage=100 instead."),
    ("{NEW_GOLD_CARRYOVER_NOTE_40}", "This is now automatically generated and not required. Use [objectives][gold_carryover]carryover_percentage=40 instead."),
    ("{NEW_GOLD_CARRYOVER_NOTE_20}", "This is now automatically generated and not required. Use [objectives][gold_carryover]carryover_percentage=20 instead."),
    ("{ON_SIGHTING", "Use an [event]name=sighted instead."),
    ("{MAKE_AI_SIDE_PERSISTENT}", "Just put the persistent key in [side] directly"),
)

# Global changes meant to be done on all lines, excluding lines containing translation mark.  Suppressed by noconvert.
linechanges = (
        ("canrecruit=1", "canrecruit=yes"),
        ("canrecruit=0", "canrecruit=no"),
        # Fix a common typo
        ("agression=", "aggression="),
        # These changed just after 1.5.0
        ("[special_filter]", "[filter_attack]"),
        ("[wml_filter]", "[filter_wml]"),
        ("[unit_filter]", "[filter]"),
        ("[secondary_unit_filter]", "[filter_second]"),
        ("[attack_filter]", "[filter_attack]"),
        ("[secondary_attack_filter]", "[filter_second_attack]"),
        ("[special_filter_second]", "[filter_second_attack]"),
        ("[/special_filter]", "[/filter_attack]"),
        ("[/wml_filter]", "[/filter_wml]"),
        ("[/unit_filter]", "[/filter]"),
        ("[/secondary_unit_filter]", "[/filter_second]"),
        ("[/attack_filter]", "[/filter_attack]"),
        ("[/secondary_attack_filter]", "[/filter_second_attack]"),
        ("[/special_filter_second]", "[/filter_second_attack]"),
        ("grassland=", "flat="),
        ("tundra=", "frozen="),
        ("cavewall=", "impassable="),
        ("canyon=", "unwalkable="),
        # This changed after 1.5.2
        ("advanceto=", "advances_to="),
        # This changed after 1.5.5, to enable mechanical spellchecking
        ("sabre", "saber"),
        ("nr-sad.ogg", "sad.ogg"),
        # Changed after 1.5.7
        ("[debug_message]", "[wml_message]"),
        ("[/debug_message]", "[/wml_message]"),
        # Changed just before 1.5.9
        ("portraits/Alex_Jarocha-Ernst/drake-burner.png",
         "portraits/drakes/burner.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-clasher.png",
         "portraits/drakes/clasher.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-fighter.png",
         "portraits/drakes/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/drake-glider.png",
         "portraits/drakes/glider.png"),
        ("portraits/Alex_Jarocha-Ernst/ghoul.png",
         "portraits/undead/ghoul.png"),
        ("portraits/Alex_Jarocha-Ernst/mermaid-initiate.png",
         "portraits/merfolk/initiate.png"),
        ("portraits/Alex_Jarocha-Ernst/merman-fighter.png",
         "portraits/merfolk/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/merman-hunter.png",
         "portraits/merfolk/hunter.png"),
        ("portraits/Alex_Jarocha-Ernst/naga-fighter.png",
         "portraits/nagas/fighter.png"),
        ("portraits/Alex_Jarocha-Ernst/nagini-fighter.png",
         "portraits/nagas/fighter+female.png"),
        ("portraits/Alex_Jarocha-Ernst/orcish-assassin.png",
         "portraits/orcs/assassin.png"),
        ("portraits/Emilien_Rotival/human-general.png",
         "portraits/humans/general.png"),
        ("portraits/Emilien_Rotival/human-heavyinfantry.png",
         "portraits/humans/heavy-infantry.png"),
        ("portraits/Emilien_Rotival/human-ironmauler.png",
         "portraits/humans/iron-mauler.png"),
        ("portraits/Emilien_Rotival/human-lieutenant.png",
         "portraits/humans/lieutenant.png"),
        ("portraits/Emilien_Rotival/human-marshal.png",
         "portraits/humans/marshal.png"),
        ("portraits/Emilien_Rotival/human-peasant.png",
         "portraits/humans/peasant.png"),
        ("portraits/Emilien_Rotival/human-pikeman.png",
         "portraits/humans/pikeman.png"),
        ("portraits/Emilien_Rotival/human-royalguard.png",
         "portraits/humans/royal-guard.png"),
        ("portraits/Emilien_Rotival/human-sergeant.png",
         "portraits/humans/sergeant.png"),
        ("portraits/Emilien_Rotival/human-spearman.png",
         "portraits/humans/spearman.png"),
        ("portraits/Emilien_Rotival/human-swordsman.png",
         "portraits/humans/swordsman.png"),
        ("portraits/Emilien_Rotival/transparent/human-general.png",
         "portraits/humans/transparent/general.png"),
        ("portraits/Emilien_Rotival/transparent/human-heavyinfantry.png",
         "portraits/humans/transparent/heavy-infantry.png"),
        ("portraits/Emilien_Rotival/transparent/human-ironmauler.png",
         "portraits/humans/transparent/iron-mauler.png"),
        ("portraits/Emilien_Rotival/transparent/human-lieutenant.png",
         "portraits/humans/transparent/lieutenant.png"),
        ("portraits/Emilien_Rotival/transparent/human-marshal.png",
         "portraits/humans/transparent/marshal.png"),
        ("portraits/Emilien_Rotival/transparent/human-marshal-2.png",
         "portraits/humans/transparent/marshal-2.png"),
        ("portraits/Emilien_Rotival/transparent/human-peasant.png",
         "portraits/humans/transparent/peasant.png"),
        ("portraits/Emilien_Rotival/transparent/human-pikeman.png",
         "portraits/humans/transparent/pikeman.png"),
        ("portraits/Emilien_Rotival/transparent/human-royalguard.png",
         "portraits/humans/transparent/royal-guard.png"),
        ("portraits/Emilien_Rotival/transparent/human-sergeant.png",
         "portraits/humans/transparent/sergeant.png"),
        ("portraits/Emilien_Rotival/transparent/human-spearman.png",
         "portraits/humans/transparent/spearman.png"),
        ("portraits/Emilien_Rotival/transparent/human-swordsman.png",
         "portraits/humans/transparent/swordsman.png"),
        ("portraits/James_Woo/assassin.png",
         "portraits/humans/assassin.png"),
        ("portraits/James_Woo/dwarf-guard.png",
         "portraits/dwarves/guard.png"),
        ("portraits/James_Woo/orc-warlord.png",
         "portraits/orcs/warlord.png"),
        ("portraits/James_Woo/orc-warlord2.png",
         "portraits/orcs/warlord2.png"),
        ("portraits/James_Woo/orc-warlord3.png",
         "portraits/orcs/warlord3.png"),
        ("portraits/James_Woo/orc-warlord4.png",
         "portraits/orcs/warlord4.png"),
        ("portraits/James_Woo/orc-warlord5.png",
         "portraits/orcs/warlord5.png"),
        ("portraits/James_Woo/troll.png",
         "portraits/trolls/troll.png"),
        ("portraits/Jason_Lutes/human-bandit.png",
         "portraits/humans/bandit.png"),
        ("portraits/Jason_Lutes/human-grand-knight.png",
         "portraits/humans/grand-knight.png"),
        ("portraits/Jason_Lutes/human-halberdier.png",
         "portraits/humans/halberdier.png"),
        ("portraits/Jason_Lutes/human-highwayman.png",
         "portraits/humans/highwayman.png"),
        ("portraits/Jason_Lutes/human-horseman.png",
         "portraits/humans/horseman.png"),
        ("portraits/Jason_Lutes/human-javelineer.png",
         "portraits/humans/javelineer.png"),
        ("portraits/Jason_Lutes/human-knight.png",
         "portraits/humans/knight.png"),
        ("portraits/Jason_Lutes/human-lancer.png",
         "portraits/humans/lancer.png"),
        ("portraits/Jason_Lutes/human-paladin.png",
         "portraits/humans/paladin.png"),
        ("portraits/Jason_Lutes/human-thug.png",
         "portraits/humans/thug.png"),
        ("portraits/Kitty/elvish-archer.png",
         "portraits/elves/archer.png"),
        ("portraits/Kitty/elvish-archer+female.png",
         "portraits/elves/archer+female.png"),
        ("portraits/Kitty/elvish-captain.png",
         "portraits/elves/captain.png"),
        ("portraits/Kitty/elvish-druid.png",
         "portraits/elves/druid.png"),
        ("portraits/Kitty/elvish-fighter.png",
         "portraits/elves/fighter.png"),
        ("portraits/Kitty/elvish-hero.png",
         "portraits/elves/hero.png"),
        ("portraits/Kitty/elvish-high-lord.png",
         "portraits/elves/high-lord.png"),
        ("portraits/Kitty/elvish-lady.png",
         "portraits/elves/lady.png"),
        ("portraits/Kitty/elvish-lord.png",
         "portraits/elves/lord.png"),
        ("portraits/Kitty/elvish-marksman.png",
         "portraits/elves/marksman.png"),
        ("portraits/Kitty/elvish-marksman+female.png",
         "portraits/elves/marksman+female.png"),
        ("portraits/Kitty/elvish-ranger.png",
         "portraits/elves/ranger.png"),
        ("portraits/Kitty/elvish-ranger+female.png",
         "portraits/elves/ranger+female.png"),
        ("portraits/Kitty/elvish-scout.png",
         "portraits/elves/scout.png"),
        ("portraits/Kitty/elvish-shaman.png",
         "portraits/elves/shaman.png"),
        ("portraits/Kitty/elvish-shyde.png",
         "portraits/elves/shyde.png"),
        ("portraits/Kitty/elvish-sorceress.png",
         "portraits/elves/sorceress.png"),
        ("portraits/Kitty/human-dark-adept.png",
         "portraits/humans/dark-adept.png"),
        ("portraits/Kitty/human-dark-adept+female.png",
         "portraits/humans/dark-adept+female.png"),
        ("portraits/Kitty/human-mage.png",
         "portraits/humans/mage.png"),
        ("portraits/Kitty/human-mage+female.png",
         "portraits/humans/mage+female.png"),
        ("portraits/Kitty/human-mage-arch.png",
         "portraits/humans/mage-arch.png"),
        ("portraits/Kitty/human-mage-arch+female.png",
         "portraits/humans/mage-arch+female.png"),
        ("portraits/Kitty/human-mage-light.png",
         "portraits/humans/mage-light.png"),
        ("portraits/Kitty/human-mage-light+female.png",
         "portraits/humans/mage-light+female.png"),
        ("portraits/Kitty/human-mage-red.png",
         "portraits/humans/mage-red.png"),
        ("portraits/Kitty/human-mage-red+female.png",
         "portraits/humans/mage-red+female.png"),
        ("portraits/Kitty/human-mage-silver.png",
         "portraits/humans/mage-silver.png"),
        ("portraits/Kitty/human-mage-silver+female.png",
         "portraits/humans/mage-silver+female.png"),
        ("portraits/Kitty/human-mage-white.png",
         "portraits/humans/mage-white.png"),
        ("portraits/Kitty/human-mage-white+female.png",
         "portraits/humans/mage-white+female.png"),
        ("portraits/Kitty/human-necromancer.png",
         "portraits/humans/necromancer.png"),
        ("portraits/Kitty/human-necromancer+female.png",
         "portraits/humans/necromancer+female.png"),
        ("portraits/Kitty/troll-whelp.png",
         "portraits/trolls/whelp.png"),
        ("portraits/Kitty/undead-lich.png",
         "portraits/undead/lich.png"),
        ("portraits/Kitty/transparent/elvish-archer.png",
         "portraits/elves/transparent/archer.png"),
        ("portraits/Kitty/transparent/elvish-archer+female.png",
         "portraits/elves/transparent/archer+female.png"),
        ("portraits/Kitty/transparent/elvish-captain.png",
         "portraits/elves/transparent/captain.png"),
        ("portraits/Kitty/transparent/elvish-druid.png",
         "portraits/elves/transparent/druid.png"),
        ("portraits/Kitty/transparent/elvish-fighter.png",
         "portraits/elves/transparent/fighter.png"),
        ("portraits/Kitty/transparent/elvish-hero.png",
         "portraits/elves/transparent/hero.png"),
        ("portraits/Kitty/transparent/elvish-high-lord.png",
         "portraits/elves/transparent/high-lord.png"),
        ("portraits/Kitty/transparent/elvish-lady.png",
         "portraits/elves/transparent/lady.png"),
        ("portraits/Kitty/transparent/elvish-lord.png",
         "portraits/elves/transparent/lord.png"),
        ("portraits/Kitty/transparent/elvish-marksman.png",
         "portraits/elves/transparent/marksman.png"),
        ("portraits/Kitty/transparent/elvish-marksman+female.png",
         "portraits/elves/transparent/marksman+female.png"),
        ("portraits/Kitty/transparent/elvish-ranger.png",
         "portraits/elves/transparent/ranger.png"),
        ("portraits/Kitty/transparent/elvish-ranger+female.png",
         "portraits/elves/transparent/ranger+female.png"),
        ("portraits/Kitty/transparent/elvish-scout.png",
         "portraits/elves/transparent/scout.png"),
        ("portraits/Kitty/transparent/elvish-shaman.png",
         "portraits/elves/transparent/shaman.png"),
        ("portraits/Kitty/transparent/elvish-shyde.png",
         "portraits/elves/transparent/shyde.png"),
        ("portraits/Kitty/transparent/elvish-sorceress.png",
         "portraits/elves/transparent/sorceress.png"),
        ("portraits/Kitty/transparent/human-dark-adept.png",
         "portraits/humans/transparent/dark-adept.png"),
        ("portraits/Kitty/transparent/human-dark-adept+female.png",
         "portraits/humans/transparent/dark-adept+female.png"),
        ("portraits/Kitty/transparent/human-mage.png",
         "portraits/humans/transparent/mage.png"),
        ("portraits/Kitty/transparent/human-mage+female.png",
         "portraits/humans/transparent/mage+female.png"),
        ("portraits/Kitty/transparent/human-mage-arch.png",
         "portraits/humans/transparent/mage-arch.png"),
        ("portraits/Kitty/transparent/human-mage-arch+female.png",
         "portraits/humans/transparent/mage-arch+female.png"),
        ("portraits/Kitty/transparent/human-mage-light.png",
         "portraits/humans/transparent/mage-light.png"),
        ("portraits/Kitty/transparent/human-mage-light+female.png",
         "portraits/humans/transparent/mage-light+female.png"),
        ("portraits/Kitty/transparent/human-mage-red.png",
         "portraits/humans/transparent/mage-red.png"),
        ("portraits/Kitty/transparent/human-mage-red+female.png",
         "portraits/humans/transparent/mage-red+female.png"),
        ("portraits/Kitty/transparent/human-mage-silver.png",
         "portraits/humans/transparent/mage-silver.png"),
        ("portraits/Kitty/transparent/human-mage-silver+female.png",
         "portraits/humans/transparent/mage-silver+female.png"),
        ("portraits/Kitty/transparent/human-mage-white.png",
         "portraits/humans/transparent/mage-white.png"),
        ("portraits/Kitty/transparent/human-mage-white+female.png",
         "portraits/humans/transparent/mage-white+female.png"),
        ("portraits/Kitty/transparent/human-necromancer.png",
         "portraits/humans/transparent/necromancer.png"),
        ("portraits/Kitty/transparent/human-necromancer+female.png",
         "portraits/humans/transparent/necromancer+female.png"),
        ("portraits/Kitty/transparent/troll-whelp.png",
         "portraits/trolls/transparent/whelp.png"),
        ("portraits/Kitty/transparent/undead-lich.png",
         "portraits/undead/transparent/lich.png"),
        ("portraits/Nicholas_Kerpan/human-poacher.png",
         "portraits/humans/poacher.png"),
        ("portraits/Nicholas_Kerpan/human-thief.png",
         "portraits/humans/thief.png"),
        ("portraits/Other/brown-lich.png",
         "portraits/undead/brown-lich.png"),
        ("portraits/Other/cavalryman.png",
         "portraits/humans/cavalryman.png"),
        ("portraits/Other/human-masterbowman.png",
         "portraits/humans/master-bowman.png"),
        ("portraits/Other/scorpion.png",
         "portraits/monsters/scorpion.png"),
        ("portraits/Other/sea-serpent.png",
         "portraits/monsters/sea-serpent.png"),
        ("portraits/Pekka_Aikio/human-bowman.png",
         "portraits/humans/bowman.png"),
        ("portraits/Pekka_Aikio/human-longbowman.png",
         "portraits/humans/longbowman.png"),
        ("portraits/Philip_Barber/dwarf-dragonguard.png",
         "portraits/dwarves/dragonguard.png"),
        ("portraits/Philip_Barber/dwarf-fighter.png",
         "portraits/dwarves/fighter.png"),
        ("portraits/Philip_Barber/dwarf-lord.png",
         "portraits/dwarves/lord.png"),
        ("portraits/Philip_Barber/dwarf-thunderer.png",
         "portraits/dwarves/thunderer.png"),
        ("portraits/Philip_Barber/saurian-augur.png",
         "portraits/saurians/augur.png"),
        ("portraits/Philip_Barber/saurian-skirmisher.png",
         "portraits/saurians/skirmisher.png"),
        ("portraits/Philip_Barber/undead-death-knight.png",
         "portraits/undead/death-knight.png"),
        ("portraits/Philip_Barber/transparent/dwarf-dragonguard.png",
         "portraits/dwarves/transparent/dragonguard.png"),
        ("portraits/Philip_Barber/transparent/dwarf-fighter.png",
         "portraits/dwarves/transparent/fighter.png"),
        ("portraits/Philip_Barber/transparent/dwarf-lord.png",
         "portraits/dwarves/transparent/lord.png"),
        ("portraits/Philip_Barber/transparent/dwarf-thunderer.png",
         "portraits/dwarves/transparent/thunderer.png"),
        ("portraits/Philip_Barber/transparent/saurian-augur.png",
         "portraits/saurians/transparent/augur.png"),
        ("portraits/Philip_Barber/transparent/saurian-skirmisher.png",
         "portraits/saurians/transparent/skirmisher.png"),
        ("portraits/Philip_Barber/transparent/undead-death-knight.png",
         "portraits/undead/transparent/death-knight.png"),
        # Changed just before 1.5.11
        ("titlescreen/landscapebattlefield.jpg",
         "story/landscape-battlefield.jpg"),
        ("titlescreen/landscapebridge.jpg",
         "story/landscape-bridge.jpg"),
        ("titlescreen/landscapecastle.jpg",
         "story/landscape-castle.jpg"),
        ("LABEL_PERSISTANT", "LABEL_PERSISTENT"),
        # Changed just before 1.5.13
        ("targetting", "targeting"),
        # Changed just after 1.7 fork
        ("[stone]", "[petrify]"),
        ("[unstone]", "[unpetrify]"),
        ("[/stone]", "[/petrify]"),
        ("[/unstone]", "[/unpetrify]"),
        ("WEAPON_SPECIAL_STONE", "WEAPON_SPECIAL_PETRIFY"),
        ("SPECIAL_NOTE_STONE", "SPECIAL_NOTE_PETRIFY"),
        (".stoned", ".petrified"),
        ("stoned=", "petrified="),
        # Changed at rev 37390
        ("swing=", "value_second="),
        # Changed just before 1.7.3
        ("Drake Gladiator", "Drake Thrasher"),
        ("units/drakes/gladiator-", "units/drakes/thrasher-"),
        ("Drake Slasher", "Drake Arbiter"),
        ("units/drakes/slasher-", "units/drakes/arbiter-"),
        # Changes after 1.7.5
        ("portraits/nagas/fighter+female.png", "portraits/nagas/fighter.png"),
        # Changes after 1.8rc1
        ("portraits/orcs/warlord.png", "portraits/orcs/transparent/warlord.png"),
        #("portraits/orcs/warlord2.png","portraits/orcs/transparent/warlord.png"), // see 1.11.5
        ("portraits/orcs/warlord3.png","portraits/orcs/transparent/grunt-2.png"),
        #("portraits/orcs/warlord4.png","portraits/orcs/transparent/grunt-2.png"), // see 1.11.5
        ("portraits/orcs/warlord5.png","portraits/orcs/transparent/grunt-3.png"),
        # Changes just before 1.9.0
        ("flat/grass-r8", "flat/grass6"),
        ("flat/grass-r7", "flat/grass5"),
        ("flat/grass-r6", "flat/grass6"),
        ("flat/grass-r5", "flat/grass5"),
        ("flat/grass-r4", "flat/grass4"),
        ("flat/grass-r3", "flat/grass3"),
        ("flat/grass-r2", "flat/grass2"),
        ("flat/grass-r1", "flat/grass1"),
        ("second_value=", "value_second="), # Correct earlier wmllint error
        (".stones", ".petrifies"),
        ("stones=", "petrifies="),
        # Changes just before 1.9.1
        ("[colour_adjust]", "[color_adjust]"),
        ("[/colour_adjust]", "[/color_adjust]"),
        ("colour=", "color="),
        ("colour_lock=", "color_lock="),
        # Changes just before 1.9.2
        ("[removeitem]", "[remove_item]"),
        ("[/removeitem]", "[/remove_item]"),
        # Changes introduced in the 1.9.x cycle
        ("TRANSITION_BASE_Ls", "TRANSITION_RESTRICTED_Ls"),
        ("TRANSITION_BASE_LF", "TRANSITION_RESTRICTED_LF"),
        ("TRANSITION_BASE_LB", "TRANSITION_RESTRICTED_LB"),
        ("TRANSITION_BASE_L", "TRANSITION_RESTRICTED_L"),
        # Changes just before 1.11.0
        ("viewing_side", "side"),
        ("duration=level", "duration=scenario"), # Note: this may be removed after 1.11.2, so an actual duration=level can be implemented
        # Changed before 1.11.5 to incorporate 1.9.0 portraits
        ("portraits/orcs/warlord2.png","portraits/orcs/transparent/grunt-5.png"),
        ("portraits/orcs/warlord4.png","portraits/orcs/transparent/grunt-6.png"),

        # Changed before 1.11.8
        ("misc/schedule-dawn.png","misc/time-schedules/default/schedule-dawn.png"),
        ("misc/schedule-morning.png","misc/time-schedules/default/schedule-morning.png"),
        ("misc/schedule-afternoon.png","misc/time-schedules/default/schedule-afternoon.png"),
        ("misc/schedule-dusk.png","misc/time-schedules/default/schedule-dusk.png"),
        ("misc/schedule-firstwatch.png","misc/time-schedules/default/schedule-firstwatch.png"),
        ("misc/schedule-secondwatch.png","misc/time-schedules/default/schedule-secondwatch.png"),

        ("misc/schedule-indoors.png","misc/time-schedules/schedule-indoors.png"),
        ("misc/schedule-underground.png","misc/time-schedules/schedule-underground.png"),
        ("misc/schedule-underground-illum.png","misc/time-schedules/schedule-underground-illum.png"),

        ("misc/tod-schedule-24hrs.png","misc/time-schedules/tod-schedule-24hrs.png"),

        # Changed before 1.13.0 to fix frames for ragged flags
        ('FLAG_VARIANT ragged','FLAG_VARIANT6 ragged'),
        ('FLAG_VARIANT "ragged"','FLAG_VARIANT6 ragged'),

        # Changed in 1.11.15.
        ("fight_on_without_leader=yes","defeat_condition=no_units_left"),
        ("fight_on_without_leader=no","defeat_condition=no_leader_left"),
        ("remove_from_carryover_on_leaders_loss=yes","remove_from_carryover_on_defeat=yes"),
        ("remove_from_carryover_on_leaders_loss=no","remove_from_carryover_on_defeat=no"),

        # Changed in 1.13.2.
        ("[advance]","[advancement]"),
        ("[/advance]","[/advancement]"),
        ("{ABILITY_LEADERSHIP_LEVEL_1}", "{ABILITY_LEADERSHIP}"),
        ("{ABILITY_LEADERSHIP_LEVEL_2}", "{ABILITY_LEADERSHIP}"),
        ("{ABILITY_LEADERSHIP_LEVEL_3}", "{ABILITY_LEADERSHIP}"),
        ("{ABILITY_LEADERSHIP_LEVEL_4}", "{ABILITY_LEADERSHIP}"),
        ("{ABILITY_LEADERSHIP_LEVEL_5}", "{ABILITY_LEADERSHIP}"),
        ("misc/icon-amla-tough.png","icons/amla-default.png"),

        # Changed in 1.13.4: removal of small portraits with black background
        ("portraits/drakes/transparent/blademaster.png", "portraits/drakes/blademaster.png"),
        ("portraits/drakes/transparent/burner.png", "portraits/drakes/burner.png"),
        ("portraits/drakes/transparent/clasher.png", "portraits/drakes/clasher.png"),
        ("portraits/drakes/transparent/enforcer.png", "portraits/drakes/enforcer.png"),
        ("portraits/drakes/transparent/fighter.png", "portraits/drakes/fighter.png"),
        ("portraits/drakes/transparent/flameheart.png", "portraits/drakes/flameheart.png"),
        ("portraits/drakes/transparent/glider.png", "portraits/drakes/glider.png"),
        ("portraits/drakes/transparent/hurricane.png", "portraits/drakes/hurricane.png"),
        ("portraits/drakes/transparent/inferno.png", "portraits/drakes/inferno.png"),
        ("portraits/drakes/transparent/warden.png", "portraits/drakes/warden.png"),
        ("portraits/dwarves/transparent/dragonguard.png", "portraits/dwarves/dragonguard.png"),
        ("portraits/dwarves/transparent/explorer.png", "portraits/dwarves/explorer.png"),
        ("portraits/dwarves/transparent/fighter-2.png", "portraits/dwarves/fighter-2.png"),
        ("portraits/dwarves/transparent/fighter.png", "portraits/dwarves/fighter.png"),
        ("portraits/dwarves/transparent/gryphon-rider.png", "portraits/dwarves/gryphon-rider.png"),
        ("portraits/dwarves/transparent/guard.png", "portraits/dwarves/guard.png"),
        ("portraits/dwarves/transparent/lord.png", "portraits/dwarves/lord.png"),
        ("portraits/dwarves/transparent/runemaster.png", "portraits/dwarves/runemaster.png"),
        ("portraits/dwarves/transparent/scout.png", "portraits/dwarves/scout.png"),
        ("portraits/dwarves/transparent/sentinel.png", "portraits/dwarves/sentinel.png"),
        ("portraits/dwarves/transparent/thunderer.png", "portraits/dwarves/thunderer.png"),
        ("portraits/dwarves/transparent/ulfserker.png", "portraits/dwarves/ulfserker.png"),
        ("portraits/elves/transparent/archer+female.png", "portraits/elves/archer+female.png"),
        ("portraits/elves/transparent/archer.png", "portraits/elves/archer.png"),
        ("portraits/elves/transparent/captain.png", "portraits/elves/captain.png"),
        ("portraits/elves/transparent/druid.png", "portraits/elves/druid.png"),
        ("portraits/elves/transparent/fighter.png", "portraits/elves/fighter.png"),
        ("portraits/elves/transparent/hero.png", "portraits/elves/hero.png"),
        ("portraits/elves/transparent/high-lord.png", "portraits/elves/high-lord.png"),
        ("portraits/elves/transparent/lady.png", "portraits/elves/lady.png"),
        ("portraits/elves/transparent/lord.png", "portraits/elves/lord.png"),
        ("portraits/elves/transparent/marksman+female.png", "portraits/elves/marksman+female.png"),
        ("portraits/elves/transparent/marksman.png", "portraits/elves/marksman.png"),
        ("portraits/elves/transparent/ranger+female.png", "portraits/elves/ranger+female.png"),
        ("portraits/elves/transparent/ranger.png", "portraits/elves/ranger.png"),
        ("portraits/elves/transparent/scout.png", "portraits/elves/scout.png"),
        ("portraits/elves/transparent/shaman.png", "portraits/elves/shaman.png"),
        ("portraits/elves/transparent/shyde.png", "portraits/elves/shyde.png"),
        ("portraits/elves/transparent/sorceress.png", "portraits/elves/sorceress.png"),
        ("portraits/elves/transparent/sylph.png", "portraits/elves/sylph.png"),
        ("portraits/goblins/transparent/direwolver.png", "portraits/goblins/direwolver.png"),
        ("portraits/goblins/transparent/impaler.png", "portraits/goblins/impaler.png"),
        ("portraits/goblins/transparent/pillager.png", "portraits/goblins/pillager.png"),
        ("portraits/goblins/transparent/rouser-2.png", "portraits/goblins/rouser-2.png"),
        ("portraits/goblins/transparent/rouser.png", "portraits/goblins/rouser.png"),
        ("portraits/goblins/transparent/spearman-2.png", "portraits/goblins/spearman-2.png"),
        ("portraits/goblins/transparent/spearman.png", "portraits/goblins/spearman.png"),
        ("portraits/goblins/transparent/wolf-rider.png", "portraits/goblins/wolf-rider.png"),
        ("portraits/humans/transparent/assassin+female.png", "portraits/humans/assassin+female.png"),
        ("portraits/humans/transparent/assassin.png", "portraits/humans/assassin.png"),
        ("portraits/humans/transparent/bandit.png", "portraits/humans/bandit.png"),
        ("portraits/humans/transparent/bowman.png", "portraits/humans/bowman.png"),
        ("portraits/humans/transparent/cavalier.png", "portraits/humans/cavalier.png"),
        ("portraits/humans/transparent/cavalryman.png", "portraits/humans/cavalryman.png"),
        ("portraits/humans/transparent/dark-adept+female.png", "portraits/humans/dark-adept+female.png"),
        ("portraits/humans/transparent/dark-adept.png", "portraits/humans/dark-adept.png"),
        ("portraits/humans/transparent/duelist.png", "portraits/humans/duelist.png"),
        ("portraits/humans/transparent/fencer.png", "portraits/humans/fencer.png"),
        ("portraits/humans/transparent/footpad+female.png", "portraits/humans/footpad+female.png"),
        ("portraits/humans/transparent/footpad.png", "portraits/humans/footpad.png"),
        ("portraits/humans/transparent/general.png", "portraits/humans/general.png"),
        ("portraits/humans/transparent/grand-knight-2.png", "portraits/humans/grand-knight-2.png"),
        ("portraits/humans/transparent/grand-knight.png", "portraits/humans/grand-knight.png"),
        ("portraits/humans/transparent/halberdier.png", "portraits/humans/halberdier.png"),
        ("portraits/humans/transparent/heavy-infantry.png", "portraits/humans/heavy-infantry.png"),
        ("portraits/humans/transparent/horseman.png", "portraits/humans/horseman.png"),
        ("portraits/humans/transparent/huntsman.png", "portraits/humans/huntsman.png"),
        ("portraits/humans/transparent/iron-mauler.png", "portraits/humans/iron-mauler.png"),
        ("portraits/humans/transparent/javelineer.png", "portraits/humans/javelineer.png"),
        ("portraits/humans/transparent/knight.png", "portraits/humans/knight.png"),
        ("portraits/humans/transparent/lancer.png", "portraits/humans/lancer.png"),
        ("portraits/humans/transparent/lieutenant.png", "portraits/humans/lieutenant.png"),
        ("portraits/humans/transparent/longbowman.png", "portraits/humans/longbowman.png"),
        ("portraits/humans/transparent/mage-arch+female.png", "portraits/humans/mage-arch+female.png"),
        ("portraits/humans/transparent/mage-arch.png", "portraits/humans/mage-arch.png"),
        ("portraits/humans/transparent/mage+female.png", "portraits/humans/mage+female.png"),
        ("portraits/humans/transparent/mage-light+female.png", "portraits/humans/mage-light+female.png"),
        ("portraits/humans/transparent/mage-light.png", "portraits/humans/mage-light.png"),
        ("portraits/humans/transparent/mage.png", "portraits/humans/mage.png"),
        ("portraits/humans/transparent/mage-red+female.png", "portraits/humans/mage-red+female.png"),
        ("portraits/humans/transparent/mage-red.png", "portraits/humans/mage-red.png"),
        ("portraits/humans/transparent/mage-silver+female.png", "portraits/humans/mage-silver+female.png"),
        ("portraits/humans/transparent/mage-silver.png", "portraits/humans/mage-silver.png"),
        ("portraits/humans/transparent/mage-white+female.png", "portraits/humans/mage-white+female.png"),
        ("portraits/humans/transparent/mage-white.png", "portraits/humans/mage-white.png"),
        ("portraits/humans/transparent/marshal-2.png", "portraits/humans/marshal-2.png"),
        ("portraits/humans/transparent/marshal.png", "portraits/humans/marshal.png"),
        ("portraits/humans/transparent/master-at-arms.png", "portraits/humans/master-at-arms.png"),
        ("portraits/humans/transparent/master-bowman.png", "portraits/humans/master-bowman.png"),
        ("portraits/humans/transparent/necromancer+female.png", "portraits/humans/necromancer+female.png"),
        ("portraits/humans/transparent/necromancer.png", "portraits/humans/necromancer.png"),
        ("portraits/humans/transparent/outlaw+female.png", "portraits/humans/outlaw+female.png"),
        ("portraits/humans/transparent/outlaw.png", "portraits/humans/outlaw.png"),
        ("portraits/humans/transparent/paladin.png", "portraits/humans/paladin.png"),
        ("portraits/humans/transparent/peasant.png", "portraits/humans/peasant.png"),
        ("portraits/humans/transparent/pikeman.png", "portraits/humans/pikeman.png"),
        ("portraits/humans/transparent/ranger.png", "portraits/humans/ranger.png"),
        ("portraits/humans/transparent/royal-guard.png", "portraits/humans/royal-guard.png"),
        ("portraits/humans/transparent/ruffian.png", "portraits/humans/ruffian.png"),
        ("portraits/humans/transparent/sergeant.png", "portraits/humans/sergeant.png"),
        ("portraits/humans/transparent/spearman-2.png", "portraits/humans/spearman-2.png"),
        ("portraits/humans/transparent/spearman.png", "portraits/humans/spearman.png"),
        ("portraits/humans/transparent/swordsman-2.png", "portraits/humans/swordsman-2.png"),
        ("portraits/humans/transparent/swordsman-3.png", "portraits/humans/swordsman-3.png"),
        ("portraits/humans/transparent/swordsman.png", "portraits/humans/swordsman.png"),
        ("portraits/humans/transparent/thief+female.png", "portraits/humans/thief+female.png"),
        ("portraits/humans/transparent/thief.png", "portraits/humans/thief.png"),
        ("portraits/humans/transparent/thug.png", "portraits/humans/thug.png"),
        ("portraits/humans/transparent/trapper.png", "portraits/humans/trapper.png"),
        ("portraits/humans/transparent/woodsman.png", "portraits/humans/woodsman.png"),
        ("portraits/khalifate/transparent/hakim.png", "portraits/khalifate/hakim.png"),
        ("portraits/merfolk/transparent/enchantress.png", "portraits/merfolk/enchantress.png"),
        ("portraits/merfolk/transparent/fighter.png", "portraits/merfolk/fighter.png"),
        ("portraits/merfolk/transparent/hoplite.png", "portraits/merfolk/hoplite.png"),
        ("portraits/merfolk/transparent/hunter.png", "portraits/merfolk/hunter.png"),
        ("portraits/merfolk/transparent/initiate-2.png", "portraits/merfolk/initiate-2.png"),
        ("portraits/merfolk/transparent/initiate.png", "portraits/merfolk/initiate.png"),
        ("portraits/merfolk/transparent/netcaster.png", "portraits/merfolk/netcaster.png"),
        ("portraits/merfolk/transparent/priestess.png", "portraits/merfolk/priestess.png"),
        ("portraits/merfolk/transparent/spearman.png", "portraits/merfolk/spearman.png"),
        ("portraits/merfolk/transparent/triton.png", "portraits/merfolk/triton.png"),
        ("portraits/monsters/transparent/bat.png", "portraits/monsters/bat.png"),
        ("portraits/monsters/transparent/deep-tentacle.png", "portraits/monsters/deep-tentacle.png"),
        ("portraits/monsters/transparent/giant-mudcrawler.png", "portraits/monsters/giant-mudcrawler.png"),
        ("portraits/monsters/transparent/gryphon.png", "portraits/monsters/gryphon.png"),
        ("portraits/monsters/transparent/ogre.png", "portraits/monsters/ogre.png"),
        ("portraits/monsters/transparent/scorpion.png", "portraits/monsters/scorpion.png"),
        ("portraits/monsters/transparent/sea-serpent.png", "portraits/monsters/sea-serpent.png"),
        ("portraits/monsters/transparent/yeti.png", "portraits/monsters/yeti.png"),
        ("portraits/monsters/transparent/young-ogre.png", "portraits/monsters/young-ogre.png"),
        ("portraits/nagas/transparent/fighter.png", "portraits/nagas/fighter.png"),
        ("portraits/nagas/transparent/myrmidon.png", "portraits/nagas/myrmidon.png"),
        ("portraits/orcs/transparent/archer.png", "portraits/orcs/archer.png"),
        ("portraits/orcs/transparent/assassin.png", "portraits/orcs/assassin.png"),
        ("portraits/orcs/transparent/crossbowman.png", "portraits/orcs/crossbowman.png"),
        ("portraits/orcs/transparent/grunt-2.png", "portraits/orcs/grunt-2.png"),
        ("portraits/orcs/transparent/grunt-3.png", "portraits/orcs/grunt-3.png"),
        ("portraits/orcs/transparent/grunt-4.png", "portraits/orcs/grunt-4.png"),
        ("portraits/orcs/transparent/grunt-5.png", "portraits/orcs/grunt-5.png"),
        ("portraits/orcs/transparent/grunt-6.png", "portraits/orcs/grunt-6.png"),
        ("portraits/orcs/transparent/grunt.png", "portraits/orcs/grunt.png"),
        ("portraits/orcs/transparent/leader-2.png", "portraits/orcs/ruler-2.png"),
        ("portraits/orcs/transparent/leader.png", "portraits/orcs/ruler.png"),
        ("portraits/orcs/transparent/slayer.png", "portraits/orcs/slayer.png"),
        ("portraits/orcs/transparent/slurbow.png", "portraits/orcs/slurbow.png"),
        ("portraits/orcs/transparent/sovereign.png", "portraits/orcs/sovereign.png"),
        ("portraits/orcs/transparent/warlord.png", "portraits/orcs/warlord.png"),
        ("portraits/orcs/transparent/warrior.png", "portraits/orcs/warrior.png"),
        ("portraits/saurians/transparent/augur.png", "portraits/saurians/augur.png"),
        ("portraits/saurians/transparent/skirmisher.png", "portraits/saurians/skirmisher.png"),
        ("portraits/trolls/transparent/troll-hero-alt.png", "portraits/trolls/troll-hero-alt.png"),
        ("portraits/trolls/transparent/troll-hero.png", "portraits/trolls/troll-hero.png"),
        ("portraits/trolls/transparent/troll.png", "portraits/trolls/troll.png"),
        ("portraits/trolls/transparent/troll-rocklobber.png", "portraits/trolls/troll-rocklobber.png"),
        ("portraits/trolls/transparent/troll-shaman.png", "portraits/trolls/troll-shaman.png"),
        ("portraits/trolls/transparent/troll-warrior.png", "portraits/trolls/troll-warrior.png"),
        ("portraits/trolls/transparent/whelp.png", "portraits/trolls/whelp.png"),
        ("portraits/undead/transparent/ancient-lich.png", "portraits/undead/ancient-lich.png"),
        ("portraits/undead/transparent/archer.png", "portraits/undead/archer.png"),
        ("portraits/undead/transparent/banebow.png", "portraits/undead/banebow.png"),
        ("portraits/undead/transparent/bone-shooter.png", "portraits/undead/bone-shooter.png"),
        ("portraits/undead/transparent/brown-lich.png", "portraits/undead/brown-lich.png"),
        ("portraits/undead/transparent/deathblade.png", "portraits/undead/deathblade.png"),
        ("portraits/undead/transparent/death-knight.png", "portraits/undead/death-knight.png"),
        ("portraits/undead/transparent/draug-2.png", "portraits/undead/draug-2.png"),
        ("portraits/undead/transparent/draug.png", "portraits/undead/draug.png"),
        ("portraits/undead/transparent/ghost.png", "portraits/undead/ghost.png"),
        ("portraits/undead/transparent/ghoul.png", "portraits/undead/ghoul.png"),
        ("portraits/undead/transparent/lich.png", "portraits/undead/lich.png"),
        ("portraits/undead/transparent/nightgaunt.png", "portraits/undead/nightgaunt.png"),
        ("portraits/undead/transparent/revenant.png", "portraits/undead/revenant.png"),
        ("portraits/undead/transparent/shadow.png", "portraits/undead/shadow.png"),
        ("portraits/undead/transparent/skeleton.png", "portraits/undead/skeleton.png"),
        ("portraits/undead/transparent/soulless.png", "portraits/undead/soulless.png"),
        ("portraits/undead/transparent/spectre.png", "portraits/undead/spectre.png"),
        ("portraits/undead/transparent/walking-corpse.png", "portraits/undead/walking-corpse.png"),
        ("portraits/undead/transparent/wraith.png", "portraits/undead/wraith.png"),
        ("portraits/woses/transparent/ancient-wose.png", "portraits/woses/ancient-wose.png"),
        ("portraits/woses/transparent/wose.png", "portraits/woses/wose.png"),

        # Consistency change for the Heavy Infantryman idle frames
        ("units/human-loyalists/heavy-infantry-idle-1.png", "units/human-loyalists/heavyinfantry-idle-1.png"),
        ("units/human-loyalists/heavy-infantry-idle-2.png", "units/human-loyalists/heavyinfantry-idle-2.png"),
        ("units/human-loyalists/heavy-infantry-idle-3.png", "units/human-loyalists/heavyinfantry-idle-3.png"),
        ("units/human-loyalists/heavy-infantry-idle-4.png", "units/human-loyalists/heavyinfantry-idle-4.png"),
        ("units/human-loyalists/heavy-infantry-idle-5.png", "units/human-loyalists/heavyinfantry-idle-5.png"),
        ("units/human-loyalists/heavy-infantry-idle-6.png", "units/human-loyalists/heavyinfantry-idle-6.png"),
        ("units/human-loyalists/heavy-infantry-idle-7.png", "units/human-loyalists/heavyinfantry-idle-7.png"),
        ("units/human-loyalists/heavy-infantry-idle-8.png", "units/human-loyalists/heavyinfantry-idle-8.png"),
        ("units/human-loyalists/heavy-infantry-idle-9.png", "units/human-loyalists/heavyinfantry-idle-9.png"),

        # renamed khalifate to dunefolk
        ("id=khalifate", "id=dunefolk"),
        ("movement_type=khalifatefoot", "movement_type=dunefoot"),
        ("movement_type=khalifatearmoredfoot", "movement_type=dunearmoredfoot"),
        ("movement_type=khalifatehorse", "movement_type=dunehorse"),
        ("movement_type=khalifatearmoredhorse", "movement_type=dunearmoredhorse"),
        ("race=khalifate", "race=dunefolk"),
        ("{KHALIFATE_NAMES}", "{DUNEFOLK_NAMES}"),
        ("portraits/khalifate/hakim.png", "portraits/dunefolk/hakim.png"),
        ("units/khalifate/arif.png", "units/dunefolk/arif.png"),
        ("units/khalifate/batal.png", "units/dunefolk/batal.png"),
        ("units/khalifate/elder-falcon.png", "units/dunefolk/elder-falcon.png"),
        ("units/khalifate/elder-falcon.png", "units/dunefolk/elder-falcon.png"),
        ("units/khalifate/faris.png", "units/dunefolk/faris.png"),
        ("units/khalifate/ghazi.png", "units/dunefolk/ghazi.png"),
        ("units/khalifate/hadaf.png", "units/dunefolk/hadaf.png"),
        ("units/khalifate/hakim.png", "units/dunefolk/hakim.png"),
        ("units/khalifate/jawal.png", "units/dunefolk/jawal.png"),
        ("units/khalifate/jundi.png", "units/dunefolk/jundi.png"),
        ("units/khalifate/khaiyal.png", "units/dunefolk/khaiyal.png"),
        ("units/khalifate/khalid.png", "units/dunefolk/khalid.png"),
        ("units/khalifate/mighwar.png", "units/dunefolk/mighwar.png"),
        ("units/khalifate/monawish.png", "units/dunefolk/monawish.png"),
        ("units/khalifate/mudafi.png", "units/dunefolk/mudafi.png"),
        ("units/khalifate/mufariq.png", "units/dunefolk/mufariq.png"),
        ("units/khalifate/muharib.png", "units/dunefolk/muharib.png"),
        ("units/khalifate/naffat.png", "units/dunefolk/naffat.png"),
        ("units/khalifate/qanas.png", "units/dunefolk/qanas.png"),
        ("units/khalifate/qatif-al-nar.png", "units/dunefolk/qatif-al-nar.png"),
        ("units/khalifate/rami.png", "units/dunefolk/rami.png"),
        ("units/khalifate/rasikh.png", "units/dunefolk/rasikh.png"),
        ("units/khalifate/saree.png", "units/dunefolk/saree.png"),
        ("units/khalifate/shuja.png", "units/dunefolk/shuja.png"),
        ("units/khalifate/tabib.png", "units/dunefolk/tabib.png"),
        ("units/khalifate/tineen.png", "units/dunefolk/tineen.png"),
        ("era_khalifate", "era_dunefolk"),
        ("era_khalifate_heroes", "era_dunefolk_heroes"),

        # renamed dunefolk units
        # images
        ("units/dunefolk/arif.png", "units/dunefolk/soldier.png"),
        ("units/dunefolk/batal.png", "units/dunefolk/wayfarer.png"),
        ("units/dunefolk/faris.png", "units/dunefolk/sunderer.png"),
        ("units/dunefolk/ghazi.png", "units/dunefolk/swordsman.png"),
        ("units/dunefolk/hadaf.png", "units/dunefolk/marauder.png"),
        ("units/dunefolk/hakim.png", "units/dunefolk/herbalist.png"),
        ("units/dunefolk/jawal.png", "units/dunefolk/windbolt.png"),
        ("units/dunefolk/jundi.png", "units/dunefolk/rover.png"),
        ("units/dunefolk/khaiyal.png", "units/dunefolk/piercer.png"),
        ("units/dunefolk/khalid.png", "units/dunefolk/warmaster.png"),
        ("units/dunefolk/mighwar.png", "units/dunefolk/harrier.png"),
        ("units/dunefolk/monawish.png", "units/dunefolk/skirmisher.png"),
        ("units/dunefolk/mudafi.png", "units/dunefolk/spearguard.png"),
        ("units/dunefolk/mufariq.png", "units/dunefolk/cataphract.png"),
        ("units/dunefolk/muharib.png", "units/dunefolk/explorer.png"),
        ("units/dunefolk/naffat.png", "units/dunefolk/burner.png"),
        ("units/dunefolk/qanas.png", "units/dunefolk/raider.png"),
        ("units/dunefolk/qatif-al-nar.png", "units/dunefolk/scorcher.png"),
        ("units/dunefolk/rami.png", "units/dunefolk/rider.png"),
        ("units/dunefolk/rasikh.png", "units/dunefolk/spearmaster.png"),
        ("units/dunefolk/saree.png", "units/dunefolk/horse-archer.png"),
        ("units/dunefolk/shuja.png", "units/dunefolk/blademaster.png"),
        ("units/dunefolk/tabib.png", "units/dunefolk/apothecary.png"),
        ("units/dunefolk/tineen.png", "units/dunefolk/firetrooper.png"),
        ("images/portraits/dunefolk/hakim.png", "images/portraits/dunefolk/herbalist.png"),

        # second round of renaming
        ("units/dunefolk/ranger.png", "units/dunefolk/wayfarer.png"),
        ("units/dunefolk/windrider.png", "units/dunefolk/windbolt.png"),
        ("units/dunefolk/swiftrider.png", "units/dunefolk/horse-archer.png"),
        ("units/nagas/slasher.png", "units/nagas/dirkfang.png"),
        ("units/nagas/bladewhirler.png", "units/nagas/ophidian.png"),

        # third round of renaming
        ("units/dunefolk/apothecary.png", "units/dunefolk/herbalist/apothecary.png"),
        ("units/dunefolk/blademaster.png", "units/dunefolk/soldier/blademaster.png"),
        ("units/dunefolk/burner.png", "units/dunefolk/burner/burner.png"),
        ("units/dunefolk/cataphract.png", "units/dunefolk/rider/cataphract.png"),
        ("units/dunefolk/explorer.png", "units/dunefolk/rover/explorer.png"),
        ("units/dunefolk/firetrooper.png", "units/dunefolk/burner/firetrooper.png"),
        ("units/dunefolk/harrier.png", "units/dunefolk/skirmisher/harrier.png"),
        ("units/dunefolk/herbalist.png", "units/dunefolk/herbalist/herbalist.png"),
        ("units/dunefolk/marauder.png", "units/dunefolk/rider/marauder.png"),
        ("units/dunefolk/raider.png", "units/dunefolk/rider/raider.png"),
        ("units/dunefolk/rider.png", "units/dunefolk/rider/rider.png"),
        ("units/dunefolk/rover.png", "units/dunefolk/rover/rover.png"),
        ("units/dunefolk/scorcher.png", "units/dunefolk/burner/scorcher.png"),
        ("units/dunefolk/skirmisher.png", "units/dunefolk/skirmisher/skirmisher.png"),
        ("units/dunefolk/soldier.png", "units/dunefolk/soldier/soldier.png"),
        ("units/dunefolk/spearguard.png", "units/dunefolk/soldier/spearguard.png"),
        ("units/dunefolk/spearmaster.png", "units/dunefolk/soldier/spearmaster.png"),
        ("units/dunefolk/sunderer.png", "units/dunefolk/rider/sunderer.png"),
        ("units/dunefolk/swordsman.png", "units/dunefolk/soldier/swordsman.png"),
        ("units/dunefolk/warmaster.png", "units/dunefolk/soldier/warmaster.png"),

        # unit ids
        ("Arif", "Dune Soldier"),
        ("Ghazi", "Dune Swordsman"),
        ("Shuja", "Dune Blademaster"),
        ("Khalid", "Dune Paragon"),
        ("Mudafi", "Dune Spearguard"),
        ("Rasikh", "Dune Spearmaster"),
        ("Hakim", "Dune Herbalist"),
        ("Tabib", "Dune Apothecary"),
        ("Jundi", "Dune Rover"),
        ("Monawish", "Dune Strider"),
        ("Mighwar", "Dune Harrier"),
        ("Muharib", "Dune Explorer"),
        ("Batal", "Dune Wayfarer"),
        ("Khaiyal", "Dune Rider"),
        ("Faris", "Dune Sunderer"),
        ("Mufariq", "Dune Cataphract"),
        ("Qanas", "Dune Raider"),
        ("Hadaf", "Dune Marauder"),
        ("Naffat", "Dune Burner"),
        ("Qatif-al-nar", "Dune Scorcher"),
        ("Qatif_al_nar", "Dune Scorcher"),
        ("Tineen", "Dune Firetrooper"),
        ("Rami", "Dune Rider"),
        ("Saree", "Dune Horse Archer"),
        ("Jawal", "Dune Windbolt"),

        # second round of renaming
        ("Dune Ranger", "Dune Wayfarer"),
        ("Dune Swiftrider", "Dune Horse Archer"),
        ("Dune Windrider", "Dune Windbolt"),
        ("Dune Piercer", "Dune Rider"),
        ("Naga Slasher", "Naga Dirkfang"),
        ("Naga Bladewhirler", "Naga Ophidian"),

        # monster unit ID changes during 1.15.x cycle
        ("Giant Stoat", "Frost Stoat"),

        # Changed in 1.15.0: separate portrait for leader
        ("portraits/orcs/leader.png", "portraits/orcs/ruler.png"),
        ("portraits/orcs/leader-2.png", "portraits/orcs/ruler-2.png"),

        # Changed just before 1.17.0
        # removed some duplicated images that were also part of an obsolete theme
        ("themes/unit_box/status-bg.png", "themes/legacy/status-bg.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-afternoon.png", "unit_env/schedule/ToD-day3.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-dawn.png", "unit_env/schedule/ToD-day2.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-dusk.png", "unit_env/schedule/ToD-day1.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-firstwatch.png", "unit_env/schedule/ToD-night.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-indoors.png", "unit_env/terrain/terrain-indoors.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-morning.png", "unit_env/schedule/ToD-day3.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-secondwatch.png", "unit_env/schedule/ToD-night.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-underground.png", "unit_env/terrain/terrain-underground.png"),
        ("themes/unit_box/misc/time-schedules/default/schedule-underground-illum.png", "misc/time-schedules/schedule-underground-illum.png"),
        ("themes/unit_box/misc/time-schedules/default/tod-schedule-24hrs.png", "misc/time-schedules/tod-schedule-24hrs.png"),

        # Changed in 1.17.2
        # added support for WebP images and converted an image to this format
        ("maps/background.jpg", "maps/background.webp"),

        # Changed in 1.17.3
        # convert a bunch more things to webp
        ("terrain/map-specific/isar-background-dawndusk.png", "terrain/map-specific/isar-background-dawndusk.webp"),
        ("terrain/map-specific/isar-background-night.png", "terrain/map-specific/isar-background-night.webp"),
        ("terrain/map-specific/isar-background.png", "terrain/map-specific/isar-background.webp"),
        ("maps/titlescreen.png", "maps/titlescreen.webp"),
        ("maps/wesnoth.png", "maps/wesnoth.webp"),
        ("maps/l10n/gd/wesnoth--overlay.png", "maps/l10n/gd/wesnoth--overlay.webp"),
        ("maps/l10n/gd/titlescreen--overlay.png", "maps/l10n/gd/titlescreen--overlay.webp"),
        ("maps/l10n/hu/wesnoth--overlay.png", "maps/l10n/hu/wesnoth--overlay.webp"),
        ("maps/l10n/it/wesnoth--overlay.png", "maps/l10n/it/wesnoth--overlay.webp"),
        ("maps/l10n/it/titlescreen--overlay.png", "maps/l10n/it/titlescreen--overlay.webp"),
        ("maps/l10n/fr/wesnoth--overlay.png", "maps/l10n/fr/wesnoth--overlay.webp"),
        ("maps/l10n/sr@ijekavianlatin/wesnoth--overlay.png", "maps/l10n/sr@ijekavianlatin/wesnoth--overlay.webp"),
        ("maps/l10n/pt/wesnoth--overlay.png", "maps/l10n/pt/wesnoth--overlay.webp"),
        ("maps/l10n/pt/titlescreen--overlay.png", "maps/l10n/pt/titlescreen--overlay.webp"),
        ("maps/l10n/sr/wesnoth--overlay.png", "maps/l10n/sr/wesnoth--overlay.webp"),
        ("maps/l10n/gl/titlescreen--overlay.png", "maps/l10n/gl/titlescreen--overlay.webp"),
        ("maps/l10n/sr@ijekavian/wesnoth--overlay.png", "maps/l10n/sr@ijekavian/wesnoth--overlay.webp"),
        ("maps/l10n/lt/titlescreen--overlay.png", "maps/l10n/lt/titlescreen--overlay.webp"),
        ("maps/l10n/es/wesnoth--overlay.png", "maps/l10n/es/wesnoth--overlay.webp"),
        ("maps/l10n/zh_CN/wesnoth--overlay.png", "maps/l10n/zh_CN/wesnoth--overlay.webp"),
        ("maps/l10n/zh_CN/titlescreen--overlay.png", "maps/l10n/zh_CN/titlescreen--overlay.webp"),
        ("maps/l10n/de/wesnoth--overlay.png", "maps/l10n/de/wesnoth--overlay.webp"),
        ("maps/l10n/de/titlescreen--overlay.png", "maps/l10n/de/titlescreen--overlay.webp"),
        ("maps/l10n/ru/wesnoth--overlay.png", "maps/l10n/ru/wesnoth--overlay.webp"),
        ("maps/l10n/ru/titlescreen--overlay.png", "maps/l10n/ru/titlescreen--overlay.webp"),
        ("maps/l10n/la/wesnoth--overlay.png", "maps/l10n/la/wesnoth--overlay.webp"),
        ("maps/l10n/sr@latin/wesnoth--overlay.png", "maps/l10n/sr@latin/wesnoth--overlay.webp"),
        ("story/landscape-plain.jpg", "story/landscape-plain.webp"),
        ("story/landscape-battlefield_nohumans.jpg", "story/landscape-battlefield_nohumans.webp"),
        ("story/landscape-bridge_sun.jpg", "story/landscape-bridge_sun.webp"),
        ("story/landscape-hills-01.jpg", "story/landscape-hills-01.webp"),
        ("story/landscape-mountains-01.jpg", "story/landscape-mountains-01.webp"),
        ("story/landscape-mountains-03.jpg", "story/landscape-mountains-03.webp"),
        ("story/landscape-mountains-05.jpg", "story/landscape-mountains-05.webp"),
        ("story/landscape-coast.jpg", "story/landscape-coast.webp"),
        ("story/landscape-bridge.jpg", "story/landscape-bridge.webp"),
        ("story/landscape-battlefield.jpg", "story/landscape-battlefield.webp"),
        ("story/landscape-mountains-02.jpg", "story/landscape-mountains-02.webp"),
        ("story/landscape-castle.jpg", "story/landscape-castle.webp"),
        ("story/swamp-01.jpg", "story/swamp-01.webp"),
        ("story/landscape-hills-02.jpg", "story/landscape-hills-02.webp"),
        ("story/landscape-mountains-04.jpg", "story/landscape-mountains-04.webp"),
        ("portraits/dwarves/fighter-2.png", "portraits/dwarves/fighter-2.webp"),
        ("portraits/dwarves/guard.png", "portraits/dwarves/guard.webp"),
        ("portraits/dwarves/dragonguard.png", "portraits/dwarves/dragonguard.webp"),
        ("portraits/dwarves/thunderer.png", "portraits/dwarves/thunderer.webp"),
        ("portraits/dwarves/lord.png", "portraits/dwarves/lord.webp"),
        ("portraits/dwarves/scout.png", "portraits/dwarves/scout.webp"),
        ("portraits/dwarves/explorer.png", "portraits/dwarves/explorer.webp"),
        ("portraits/dwarves/gryphon-rider.png", "portraits/dwarves/gryphon-rider.webp"),
        ("portraits/dwarves/ulfserker.png", "portraits/dwarves/ulfserker.webp"),
        ("portraits/dwarves/sentinel.png", "portraits/dwarves/sentinel.webp"),
        ("portraits/dwarves/fighter.png", "portraits/dwarves/fighter.webp"),
        ("portraits/dwarves/runemaster.png", "portraits/dwarves/runemaster.webp"),
        ("portraits/dunefolk/herbalist.png", "portraits/dunefolk/herbalist.webp"),
        ("portraits/merfolk/triton.png", "portraits/merfolk/triton.webp"),
        ("portraits/merfolk/enchantress.png", "portraits/merfolk/enchantress.webp"),
        ("portraits/merfolk/spearman.png", "portraits/merfolk/spearman.webp"),
        ("portraits/merfolk/initiate.png", "portraits/merfolk/initiate.webp"),
        ("portraits/merfolk/hunter.png", "portraits/merfolk/hunter.webp"),
        ("portraits/merfolk/priestess.png", "portraits/merfolk/priestess.webp"),
        ("portraits/merfolk/hoplite.png", "portraits/merfolk/hoplite.webp"),
        ("portraits/merfolk/netcaster.png", "portraits/merfolk/netcaster.webp"),
        ("portraits/merfolk/initiate-2.png", "portraits/merfolk/initiate-2.webp"),
        ("portraits/merfolk/fighter.png", "portraits/merfolk/fighter.webp"),
        ("portraits/undead/skeleton.png", "portraits/undead/skeleton.webp"),
        ("portraits/undead/draug-2.png", "portraits/undead/draug-2.webp"),
        ("portraits/undead/brown-lich.png", "portraits/undead/brown-lich.webp"),
        ("portraits/undead/zombie-horse.png", "portraits/undead/zombie-horse.webp"),
        ("portraits/undead/ghoul.png", "portraits/undead/ghoul.webp"),
        ("portraits/undead/zombie-drake.png", "portraits/undead/zombie-drake.webp"),
        ("portraits/undead/zombie-troll.png", "portraits/undead/zombie-troll.webp"),
        ("portraits/undead/zombie-saurian.png", "portraits/undead/zombie-saurian.webp"),
        ("portraits/undead/zombie-rider.png", "portraits/undead/zombie-rider.webp"),
        ("portraits/undead/zombie-gryphon.png", "portraits/undead/zombie-gryphon.webp"),
        ("portraits/undead/zombie-dwarf.png", "portraits/undead/zombie-dwarf.webp"),
        ("portraits/undead/lich.png", "portraits/undead/lich.webp"),
        ("portraits/undead/walking-corpse.png", "portraits/undead/walking-corpse.webp"),
        ("portraits/undead/revenant.png", "portraits/undead/revenant.webp"),
        ("portraits/undead/zombie-wose.png", "portraits/undead/zombie-wose.webp"),
        ("portraits/undead/zombie-bat.png", "portraits/undead/zombie-bat.webp"),
        ("portraits/undead/banebow.png", "portraits/undead/banebow.webp"),
        ("portraits/undead/draug.png", "portraits/undead/draug.webp"),
        ("portraits/undead/zombie-swimmer.png", "portraits/undead/zombie-swimmer.webp"),
        ("portraits/undead/death-knight.png", "portraits/undead/death-knight.webp"),
        ("portraits/undead/zombie-goblin.png", "portraits/undead/zombie-goblin.webp"),
        ("portraits/undead/zombie-rat.png", "portraits/undead/zombie-rat.webp"),
        ("portraits/undead/archer.png", "portraits/undead/archer.webp"),
        ("portraits/undead/shadow.png", "portraits/undead/shadow.webp"),
        ("portraits/undead/spectre.png", "portraits/undead/spectre.webp"),
        ("portraits/undead/zombie-boar.png", "portraits/undead/zombie-boar.webp"),
        ("portraits/undead/zombie-falcon.png", "portraits/undead/zombie-falcon.webp"),
        ("portraits/undead/zombie-spider.png", "portraits/undead/zombie-spider.webp"),
        ("portraits/undead/zombie-scorpion.png", "portraits/undead/zombie-scorpion.webp"),
        ("portraits/undead/soulless.png", "portraits/undead/soulless.webp"),
        ("portraits/undead/ghost.png", "portraits/undead/ghost.webp"),
        ("portraits/undead/deathblade.png", "portraits/undead/deathblade.webp"),
        ("portraits/undead/wraith.png", "portraits/undead/wraith.webp"),
        ("portraits/undead/zombie-beastrider.png", "portraits/undead/zombie-beastrider.webp"),
        ("portraits/undead/zombie-wolf.png", "portraits/undead/zombie-wolf.webp"),
        ("portraits/undead/ancient-lich.png", "portraits/undead/ancient-lich.webp"),
        ("portraits/undead/zombie-serpent.png", "portraits/undead/zombie-serpent.webp"),
        ("portraits/undead/bone-shooter.png", "portraits/undead/bone-shooter.webp"),
        ("portraits/undead/nightgaunt.png", "portraits/undead/nightgaunt.webp"),
        ("portraits/elves/lady.png", "portraits/elves/lady.webp"),
        ("portraits/elves/shaman.png", "portraits/elves/shaman.webp"),
        ("portraits/elves/ranger.png", "portraits/elves/ranger.webp"),
        ("portraits/elves/marksman+female.png", "portraits/elves/marksman+female.webp"),
        ("portraits/elves/sorceress.png", "portraits/elves/sorceress.webp"),
        ("portraits/elves/druid.png", "portraits/elves/druid.webp"),
        ("portraits/elves/sylph.png", "portraits/elves/sylph.webp"),
        ("portraits/elves/archer.png", "portraits/elves/archer.webp"),
        ("portraits/elves/lord.png", "portraits/elves/lord.webp"),
        ("portraits/elves/scout.png", "portraits/elves/scout.webp"),
        ("portraits/elves/archer+female.png", "portraits/elves/archer+female.webp"),
        ("portraits/elves/captain.png", "portraits/elves/captain.webp"),
        ("portraits/elves/high-lord.png", "portraits/elves/high-lord.webp"),
        ("portraits/elves/ranger+female.png", "portraits/elves/ranger+female.webp"),
        ("portraits/elves/shyde.png", "portraits/elves/shyde.webp"),
        ("portraits/elves/marksman.png", "portraits/elves/marksman.webp"),
        ("portraits/elves/hero.png", "portraits/elves/hero.webp"),
        ("portraits/elves/fighter.png", "portraits/elves/fighter.webp"),
        ("portraits/nagas/myrmidon.png", "portraits/nagas/myrmidon.webp"),
        ("portraits/nagas/dirkfang.png", "portraits/nagas/dirkfang.webp"),
        ("portraits/nagas/naga-mace_warrior2.png", "portraits/nagas/naga-mace_warrior2.webp"),
        ("portraits/nagas/naga-mace_warrior1.png", "portraits/nagas/naga-mace_warrior1.webp"),
        ("portraits/nagas/naga-ophidian.png", "portraits/nagas/naga-ophidian.webp"),
        ("portraits/nagas/naga-mace_warrior3.png", "portraits/nagas/naga-mace_warrior3.webp"),
        ("portraits/nagas/naga-ringcaster.png", "portraits/nagas/naga-ringcaster.webp"),
        ("portraits/nagas/fighter.png", "portraits/nagas/fighter.webp"),
        ("portraits/woses/wose.png", "portraits/woses/wose.webp"),
        ("portraits/orcs/grunt-5.png", "portraits/orcs/grunt-5.webp"),
        ("portraits/orcs/warlord.png", "portraits/orcs/warlord.webp"),
        ("portraits/orcs/grunt-6.png", "portraits/orcs/grunt-6.webp"),
        ("portraits/orcs/grunt.png", "portraits/orcs/grunt.webp"),
        ("portraits/orcs/archer.png", "portraits/orcs/archer.webp"),
        ("portraits/orcs/warrior.png", "portraits/orcs/warrior.webp"),
        ("portraits/orcs/ruler-2.png", "portraits/orcs/ruler-2.webp"),
        ("portraits/orcs/crossbowman.png", "portraits/orcs/crossbowman.webp"),
        ("portraits/orcs/slayer.png", "portraits/orcs/slayer.webp"),
        ("portraits/orcs/grunt-4.png", "portraits/orcs/grunt-4.webp"),
        ("portraits/orcs/grunt-2.png", "portraits/orcs/grunt-2.webp"),
        ("portraits/orcs/sovereign.png", "portraits/orcs/sovereign.webp"),
        ("portraits/orcs/assassin.png", "portraits/orcs/assassin.webp"),
        ("portraits/orcs/ruler.png", "portraits/orcs/ruler.webp"),
        ("portraits/orcs/grunt-3.png", "portraits/orcs/grunt-3.webp"),
        ("portraits/orcs/slurbow.png", "portraits/orcs/slurbow.webp"),
        ("portraits/wolves/wolf-red.png", "portraits/wolves/wolf-red.webp"),
        ("portraits/wolves/wolf-great.png", "portraits/wolves/wolf-great.webp"),
        ("portraits/wolves/wolf-chewing.png", "portraits/wolves/wolf-chewing.webp"),
        ("portraits/wolves/wolf-dark.png", "portraits/wolves/wolf-dark.webp"),
        ("portraits/humans/outlaw+female.png", "portraits/humans/outlaw+female.webp"),
        ("portraits/humans/dark-adept+female.png", "portraits/humans/dark-adept+female.webp"),
        ("portraits/humans/outlaw.png", "portraits/humans/outlaw.webp"),
        ("portraits/humans/necromancer.png", "portraits/humans/necromancer.webp"),
        ("portraits/humans/marshal.png", "portraits/humans/marshal.webp"),
        ("portraits/humans/woodsman.png", "portraits/humans/woodsman.webp"),
        ("portraits/humans/fencer.png", "portraits/humans/fencer.webp"),
        ("portraits/humans/general.png", "portraits/humans/general.webp"),
        ("portraits/humans/lancer.png", "portraits/humans/lancer.webp"),
        ("portraits/humans/bandit.png", "portraits/humans/bandit.webp"),
        ("portraits/humans/ranger.png", "portraits/humans/ranger.webp"),
        ("portraits/humans/spearman-2.png", "portraits/humans/spearman-2.webp"),
        ("portraits/humans/thief+female.png", "portraits/humans/thief+female.webp"),
        ("portraits/humans/mage-white.png", "portraits/humans/mage-white.webp"),
        ("portraits/humans/halberdier.png", "portraits/humans/halberdier.webp"),
        ("portraits/humans/peasant.png", "portraits/humans/peasant.webp"),
        ("portraits/humans/huntsman.png", "portraits/humans/huntsman.webp"),
        ("portraits/humans/pikeman.png", "portraits/humans/pikeman.webp"),
        ("portraits/humans/mage-red.png", "portraits/humans/mage-red.webp"),
        ("portraits/humans/longbowman.png", "portraits/humans/longbowman.webp"),
        ("portraits/humans/swordsman-3.png", "portraits/humans/swordsman-3.webp"),
        ("portraits/humans/mage-red+female.png", "portraits/humans/mage-red+female.webp"),
        ("portraits/humans/footpad+female.png", "portraits/humans/footpad+female.webp"),
        ("portraits/humans/spearman.png", "portraits/humans/spearman.webp"),
        ("portraits/humans/mage-white+female.png", "portraits/humans/mage-white+female.webp"),
        ("portraits/humans/grand-knight.png", "portraits/humans/grand-knight.webp"),
        ("portraits/humans/mage-arch+female.png", "portraits/humans/mage-arch+female.webp"),
        ("portraits/humans/trapper.png", "portraits/humans/trapper.webp"),
        ("portraits/humans/bowman.png", "portraits/humans/bowman.webp"),
        ("portraits/humans/horseman.png", "portraits/humans/horseman.webp"),
        ("portraits/humans/grand-knight-2.png", "portraits/humans/grand-knight-2.webp"),
        ("portraits/humans/royal-warrior.png", "portraits/humans/royal-warrior.webp"),
        ("portraits/humans/iron-mauler.png", "portraits/humans/iron-mauler.webp"),
        ("portraits/humans/mage-light.png", "portraits/humans/mage-light.webp"),
        ("portraits/humans/necromancer+female.png", "portraits/humans/necromancer+female.webp"),
        ("portraits/humans/paladin.png", "portraits/humans/paladin.webp"),
        ("portraits/humans/thief.png", "portraits/humans/thief.webp"),
        ("portraits/humans/lieutenant.png", "portraits/humans/lieutenant.webp"),
        ("portraits/humans/mage+female.png", "portraits/humans/mage+female.webp"),
        ("portraits/humans/mage.png", "portraits/humans/mage.webp"),
        ("portraits/humans/dark-adept.png", "portraits/humans/dark-adept.webp"),
        ("portraits/humans/assassin+female.png", "portraits/humans/assassin+female.webp"),
        ("portraits/humans/cavalier.png", "portraits/humans/cavalier.webp"),
        ("portraits/humans/mage-silver+female.png", "portraits/humans/mage-silver+female.webp"),
        ("portraits/humans/footpad.png", "portraits/humans/footpad.webp"),
        ("portraits/humans/knight.png", "portraits/humans/knight.webp"),
        ("portraits/humans/royal-guard.png", "portraits/humans/royal-guard.webp"),
        ("portraits/humans/assassin.png", "portraits/humans/assassin.webp"),
        ("portraits/humans/sergeant.png", "portraits/humans/sergeant.webp"),
        ("portraits/humans/mage-light+female.png", "portraits/humans/mage-light+female.webp"),
        ("portraits/humans/ruffian.png", "portraits/humans/ruffian.webp"),
        ("portraits/humans/javelineer.png", "portraits/humans/javelineer.webp"),
        ("portraits/humans/swordsman.png", "portraits/humans/swordsman.webp"),
        ("portraits/humans/thug.png", "portraits/humans/thug.webp"),
        ("portraits/humans/heavy-infantry.png", "portraits/humans/heavy-infantry.webp"),
        ("portraits/humans/master-bowman.png", "portraits/humans/master-bowman.webp"),
        ("portraits/humans/cavalryman.png", "portraits/humans/cavalryman.webp"),
        ("portraits/humans/mage-arch.png", "portraits/humans/mage-arch.webp"),
        ("portraits/humans/master-at-arms.png", "portraits/humans/master-at-arms.webp"),
        ("portraits/humans/duelist.png", "portraits/humans/duelist.webp"),
        ("portraits/humans/swordsman-2.png", "portraits/humans/swordsman-2.webp"),
        ("portraits/humans/marshal-2.png", "portraits/humans/marshal-2.webp"),
        ("portraits/humans/mage-silver.png", "portraits/humans/mage-silver.webp"),
        ("portraits/drakes/glider.png", "portraits/drakes/glider.webp"),
        ("portraits/drakes/enforcer.png", "portraits/drakes/enforcer.webp"),
        ("portraits/drakes/blademaster.png", "portraits/drakes/blademaster.webp"),
        ("portraits/drakes/flameheart.png", "portraits/drakes/flameheart.webp"),
        ("portraits/drakes/warden.png", "portraits/drakes/warden.webp"),
        ("portraits/drakes/hurricane.png", "portraits/drakes/hurricane.webp"),
        ("portraits/drakes/burner.png", "portraits/drakes/burner.webp"),
        ("portraits/drakes/clasher.png", "portraits/drakes/clasher.webp"),
        ("portraits/drakes/inferno.png", "portraits/drakes/inferno.webp"),
        ("portraits/drakes/fighter.png", "portraits/drakes/fighter.webp"),
        ("portraits/trolls/troll-hero.png", "portraits/trolls/troll-hero.webp"),
        ("portraits/trolls/troll-warrior.png", "portraits/trolls/troll-warrior.webp"),
        ("portraits/trolls/troll-hero-alt.png", "portraits/trolls/troll-hero-alt.webp"),
        ("portraits/trolls/whelp.png", "portraits/trolls/whelp.webp"),
        ("portraits/trolls/troll-rocklobber.png", "portraits/trolls/troll-rocklobber.webp"),
        ("portraits/trolls/troll-shaman.png", "portraits/trolls/troll-shaman.webp"),
        ("portraits/saurians/augur.png", "portraits/saurians/augur.webp"),
        ("portraits/saurians/skirmisher.png", "portraits/saurians/skirmisher.webp"),
        ("portraits/goblins/spearman-2.png", "portraits/goblins/spearman-2.webp"),
        ("portraits/goblins/rouser-2.png", "portraits/goblins/rouser-2.webp"),
        ("portraits/goblins/spearman.png", "portraits/goblins/spearman.webp"),
        ("portraits/goblins/direwolver.png", "portraits/goblins/direwolver.webp"),
        ("portraits/goblins/wolf-rider.png", "portraits/goblins/wolf-rider.webp"),
        ("portraits/goblins/impaler.png", "portraits/goblins/impaler.webp"),
        ("portraits/goblins/pillager.png", "portraits/goblins/pillager.webp"),
        ("portraits/goblins/rouser.png", "portraits/goblins/rouser.webp"),
        ("portraits/monsters/small-icemonax.png", "portraits/monsters/small-icemonax.webp"),
        ("portraits/monsters/bat-dread.png", "portraits/monsters/bat-dread.webp"),
        ("portraits/monsters/fire-dragon.png", "portraits/monsters/fire-dragon.webp"),
        ("portraits/monsters/fire_guardian.png", "portraits/monsters/fire_guardian.webp"),
        ("portraits/monsters/yeti.png", "portraits/monsters/yeti.webp"),
        ("portraits/monsters/scarab.png", "portraits/monsters/scarab.webp"),
        ("portraits/monsters/water-serpent.png", "portraits/monsters/water-serpent.webp"),
        ("portraits/monsters/black-stallion.png", "portraits/monsters/black-stallion.webp"),
        ("portraits/monsters/scuttler.png", "portraits/monsters/scuttler.webp"),
        ("portraits/monsters/scorpion-elder.png", "portraits/monsters/scorpion-elder.webp"),
        ("portraits/monsters/bear.png", "portraits/monsters/bear.webp"),
        ("portraits/monsters/gryphon.png", "portraits/monsters/gryphon.webp"),
        ("portraits/monsters/jinn.png", "portraits/monsters/jinn.webp"),
        ("portraits/monsters/bat-red.png", "portraits/monsters/bat-red.webp"),
        ("portraits/monsters/fire_ant.png", "portraits/monsters/fire_ant.webp"),
        ("portraits/monsters/cuttlefish.png", "portraits/monsters/cuttlefish.webp"),
        ("portraits/monsters/woodland_boar.png", "portraits/monsters/woodland_boar.webp"),
        ("portraits/monsters/seahorse.png", "portraits/monsters/seahorse.webp"),
        ("portraits/monsters/stoat.png", "portraits/monsters/stoat.webp"),
        ("portraits/monsters/scorpion.png", "portraits/monsters/scorpion.webp"),
        ("portraits/monsters/falcon.png", "portraits/monsters/falcon.webp"),
        ("portraits/monsters/mudcrawler.png", "portraits/monsters/mudcrawler.webp"),
        ("portraits/monsters/young-ogre.png", "portraits/monsters/young-ogre.webp"),
        ("portraits/monsters/giant-rat.png", "portraits/monsters/giant-rat.webp"),
        ("portraits/monsters/nightmare.png", "portraits/monsters/nightmare.webp"),
        ("portraits/monsters/bat.png", "portraits/monsters/bat.webp"),
        ("portraits/monsters/fire_wraith_A.png", "portraits/monsters/fire_wraith_A.webp"),
        ("portraits/monsters/deep-tentacle.png", "portraits/monsters/deep-tentacle.webp"),
        ("portraits/monsters/crocodile.png", "portraits/monsters/crocodile.webp"),
        ("portraits/monsters/horse.png", "portraits/monsters/horse.webp"),
        ("portraits/monsters/giant_ant.png", "portraits/monsters/giant_ant.webp"),
        ("portraits/monsters/giant-mudcrawler.png", "portraits/monsters/giant-mudcrawler.webp"),
        ("portraits/monsters/giant-spider.png", "portraits/monsters/giant-spider.webp"),
        ("portraits/monsters/scorpion-rock.png", "portraits/monsters/scorpion-rock.webp"),
        ("portraits/monsters/piglet.png", "portraits/monsters/piglet.webp"),
        ("portraits/monsters/sea-serpent.png", "portraits/monsters/sea-serpent.webp"),
        ("portraits/monsters/big-icemonax.png", "portraits/monsters/big-icemonax.webp"),
        ("portraits/monsters/ogre.png", "portraits/monsters/ogre.webp"),

        # grouping of sprites sharing a color map in 1.17.4
        # bats
        ("units/undead/bat-", "units/bats/bat-"),
        ("units/undead/bloodbat-", "units/bats/bloodbat-"),
        ("units/undead/dreadbat-", "units/bats/dreadbat-"),
        # spirits
        ("units/undead/ghost-", "units/undead-spirit/ghost-"),
        ("units/undead/wraith-", "units/undead-spirit/wraith-"),
        ("units/undead/shadow-", "units/undead-spirit/shadow-"),
        ("units/undead/spectre", "units/undead-spirit/spectre"),
        ("units/undead/nightgaunt", "units/undead-spirit/nightgaunt"),
        # nagas
        ("units/nagas/fighter.png", "units/nagas/fighter/fighter.png"),
        ("units/nagas/fighter-", "units/nagas/fighter/fighter-"),
        ("units/nagas/warrior", "units/nagas/fighter/warrior"),
        ("units/nagas/myrmidon", "units/nagas/fighter/myrmidon"),
        ("units/nagas/guardian.png", "units/nagas/guardian/guardian.png"),
        ("units/nagas/guardian-", "units/nagas/guardian/guardian-"),
        ("units/nagas/warden", "units/nagas/guardian/warden"),
        ("units/nagas/sentinel", "units/nagas/guardian/sentinel"),
        ("units/nagas/dirkfang", "units/nagas/mixed/dirkfang"),
        ("units/nagas/ringcaster", "units/nagas/mixed/ringcaster"),
        ("units/nagas/ophidian", "units/nagas/mixed/ophidian"),
        ("units/nagas/sicarius", "units/nagas/mixed/sicarius"),
        ("units/nagas/zephyr", "units/nagas/mixed/zephyr"),

        # introduced in wesnoth 1.17.19
        ("units/elves-wood/fighter.png", "units/elves-wood/fighter/fighter.png"),

        # some macro conversions from 1.19
        ("{MISSILE_FRAME_FIREBALL}",
         "{MISSILE_FRAME_FIREBALL_XY 0 0}"),
        )

# helper function to reduce code duplication and allow easy checking
# of amendment tags ([+tag] syntax)

def has_opening_tag(line, tag):
    """Check whether a line contains an opening tag or the corresponding
amendment tag, internally using string concatenation. Returns a boolean."""
    return (("[" + tag + "]") in line) or (("[+" + tag + "]") in line)

def validate_on_pop(tagstack, closer, filename, lineno):
    "Validate the stack at the time a new close tag is seen."
    (tag, attributes, subtags) = tagstack[-1]
    # append the closer to the subtags of the parent tag
    # don't do it if we're closing a root tag
    if len(tagstack) > 1:
        tagstack[-2][2].append(closer)
    ancestors = [x[0] for x in tagstack]
    if verbose >= 3:
        print('"%s", line %d: closing %s I see %s with %s' % (filename, lineno, closer, tag, attributes))
    # This assumes that conversion will always happen in units/ files.
    if "units" not in filename and closer == "unit" and "race" in attributes:
        print('"%s", line %d: [unit] needs hand fixup to [unit_type]' % \
              (filename, lineno))
    if closer in ["campaign", "race"] and "id" not in attributes:
        print('"%s", line %d: %s requires an ID attribute but has none' % \
              (filename, lineno, closer))
    if closer == "terrain" and attributes.get("heals") in ("true", "false"):
        print('"%s", line %d: heals attribute no longer takes a boolean' % \
              (filename, lineno))
    if closer == "unit" and attributes.get("id") is not None and attributes.get("type") is not None and attributes.get("side") is None and not "side" in ancestors:
        print('"%s", line %d: unit declaration without side attribute' % \
              (filename, lineno))
    if closer == "theme" and "id" not in attributes:
        if "name" in attributes:
            print('"%s", line %d: using [theme]name= instead of [theme]id= is deprecated' % (filename, lineno))
        else:
            print('"%s", line %d: [theme] needs an id attribute' % (filename, lineno))
    # Check for user-visible themes that lack a UI name or description.
    if closer == "theme" and ("hidden" not in attributes or attributes["hidden"] not in ("yes", "true")):
        for attr in ("name", "description"):
            if attr not in attributes:
                print('"%s", line %d: [theme] needs a %s attribute unless hidden=yes' % \
                      (filename, lineno, attr))
    if closer == "filter_side":
        ancestor = False
        if "gold" in ancestors:
            ancestor = "gold"
        elif "modify_ai" in ancestors:
            ancestor = "modify_ai"
        if ancestor:
            print('"%s", line %d: %s should have an inline SSF instead of using [filter_side]' % \
                  (filename, lineno, ancestor))
    if closer == "effect":
        if attributes.get("unit_type") is not None:
            print('"%s", line %d: use [effect][filter]type= instead of [effect]unit_type=' % \
                  (filename, lineno))
        if attributes.get("unit_gender") is not None:
            print('"%s", line %d: use [effect][filter]gender= instead of [effect]unit_gender=' % \
                  (filename, lineno))
    if missingside and closer in ["set_recruit", "allow_recruit", "disallow_recruit", "store_gold"] and "side" not in attributes:
        print('"%s", line %d: %s without "side" attribute is now applied to all sides' % \
              (filename, lineno, closer))
    if closer == "variation" and "variation_id" not in attributes:
        print('"%s", line %d: [variation] is missing required variation_id attribute' % \
              (filename, lineno))
    if closer == "filter" and "kill" in ancestors:
        print('"%s", line %d: [kill] should have an inline StandardUnitFilter instead of using [filter]' % \
              (filename, lineno))

def within(tag):
    "Did the specified tag lead one of our enclosing contexts?"
    if type(tag) == type(()): # Can take a list.
        for t in tag:
            if within(t):
                return True
        else:
            return False
    else:
        return tag in [x[0] for x in tagstack]

def under(tag):
    "Did the specified tag lead the latest context?"
    if type(tag) == type(()): # Can take a list.
        for t in tag:
            if within(t):
                return True
        else:
            return False
    elif tagstack:
        return tag == tagstack[-1][0]
    else:
        return False

def standard_unit_filter():
    "Are we within the syntactic context of a standard unit filter?"
    # It's under("message") rather than within("message") because
    # [message] can contain [option] markup with menu item description=
    # attributes that should not be altered.
    return within(("filter", "filter_second",
                   "filter_adjacent", "filter_opponent",
                   "unit_filter", "secondary_unit_filter",
                   "special_filter", "special_filter_second",
                   "neighbor_unit_filter",
                   "recall", "teleport", "kill", "unstone", "store_unit",
                   "have_unit", "scroll_to_unit", "role",
                   "hide_unit", "unhide_unit",
                   "protect_unit", "target", "avoid")) \
                   or under("message")

# Sanity checking

# This set of the standard recruitable usage types can be appended with the
# magic comment, "#wmllint: usagetype[s]".
standard_usage_types = {"scout", "fighter", "mixed fighter", "archer", "healer"}

# Since 1.13, UMC authors can define their own conditional tags in Lua
# This list can be populated by using the magic comment
# # wmllint: conditional tag <tag_name>
# It will be then used by local_sanity_check()
custom_conditionals = []

# These are accumulated by sanity_check() and examined by consistency_check()
unit_types = []
derived_units = []
usage = {}
sides = []
advances = []
movetypes = []
unit_movetypes = []
races = []
unit_races = []
nextrefs = []
scenario_to_filename = {}
unit_levels = {}

# Attributes that should have translation marks
def is_translatable(key, extra_translatables):
    translatables = (
        "abbrev", "base_names", "cannot_use_message", "caption", "current_player",
        "currently_doing_description", "description", "description_inactive",
        "editor_name", "end_text", "difficulty_descriptions", "female_message",
        "female_name_inactive", "female_names", "female_text", "help_text",
        "help_topic_text", "label", "male_message", "male_names", "male_text",
        "message", "name", "names", "name_inactive", "new_game_title", "note",
        "option_description", "option_name", "order", "plural_name", "prefix",
        "reason", "set_description", "source", "story", "summary", "victory_string",
        "defeat_string", "gold_carryover_string", "notes_string", "text", "title",
        "title2", "tooltip", "translator_comment", "user_team_name", "side_name",
        "special_note","display_name","name_affected","description_affected","family_order",
        "family_order_monospace","family_order_script","name_completed","description_completed",
        "lawful_description","neutral_description","chaotic_description","liminal_description",
        "hint_text","has_helptip_message","maximum_value_label","minimum_value_label","help",
        "block_dismiss_message"
        )
    return key in translatables or key in extra_translatables or (key.startswith(("type_", "range_")) and key != "type_adv_tree")

# This is a list of mainline campaigns, used to convert UMC from
# "data/campaigns" to "data/add-ons" while not clobbering mainline.
mainline = ("Dead_Water",
            "Descent_Into_Darkness",
            "Dusk_of_Dawn",
            "Eastern_Invasion",
            "Heir_To_The_Throne",
            "Heir_To_The_Throne_Classic",
            "Legend_of_Wesmere",
            "Liberty",
            "Northern_Rebirth",
            "Of_Pearls_and_Pirates",
            "Sceptre_of_Fire",
            "Secrets_of_the_Ancients",
            "Son_Of_The_Black_Eye",
            "The_Deceivers_Gambit",
            "The_Hammer_of_Thursagan",
            "The_Rise_Of_Wesnoth",
            "The_South_Guard",
            "Two_Brothers",
            "Under_the_Burning_Suns",
            "Winds_of_Fate",
            "World_Conquest",
            )

pango_conversions = (("~", "<b>", "</b>", re.compile(r"[~][a-zA-Z0-9]*[~]")),
                     ("@", "<span color='green'>", "</span>", re.compile(r"[@][a-zA-Z0-9]*[@]")),
                     ("#", "<span color='red'>", "</span>", re.compile(r"[#][a-zA-Z0-9]*[#]")),
                     ("*", "<span size='large'>", "</span>", re.compile(r"[*][a-zA-Z0-9]*[*]")),
                     ("`", "<span size='small'>", "</span>", re.compile(r"[`][a-zA-Z0-9]*[`]")),
                     )

def pangostrip(message):
    "Strip Pango markup out of a string."
    # This is all known Pango convenience tags
    for tag in ("b", "big", "i", "s", "sub", "sup", "small", "tt", "u"):
        message = message.replace("<%s>" % tag, "").replace("</%s>" % tag, "")
    # Now remove general span tags
    message = re.sub("</?span[^>]*>", "", message)
    # And Pango specials;
    message = re.sub("&[a-z]+;", "", message)
    return message

def pangoize(message, filename, line):
    "Pango conversion of old-style Wesnoth markup."
    if '&' in message:
        amper = message.find('&')
        if message[amper:amper+1].isspace():
            message = message[:amper] + "&amp;" + message[amper+1:]
    rgb = re.search("(?:<|&lt;)([0-9]+),([0-9]+),([0-9]+)(?:>|&gt;)", message)
    if rgb:
        r, g, b = (min(255, int(c)) for c in rgb.groups())
        hexed = '%02x%02x%02x' % (r, g, b)
        print('"%s", line %d: color spec (%s) requires manual fix (<span color=\'#%s\'>, </span>).' % (filename, line, rgb.group(), hexed))
    # Hack old-style Wesnoth markup
    for (oldstyle, newstart, newend, regex) in pango_conversions:
        # avoid incorrectly converting stuff like "*Ahem*"
        if oldstyle not in message or regex.search(message):
            continue
        where = message.find(oldstyle)
        if message[where - 1] != '"': # Start of string only
            continue
        if message.strip()[-1] != '"':
            print('"%s", line %d: %s highlight at start of multiline string requires manual fix.' % (filename, line, oldstyle))
            continue
        if '+' in message:
            print('"%s", line %d: %s highlight in composite string requires manual fix.' % (filename, line, oldstyle))
            continue
        # This is the common, simple case we can fix automatically
        message = message[:where] + newstart + message[where + 1:]
        endq = message.rfind('"')
        message = message[:endq] + newend + message[endq:]
    # Check for unescaped < and >
    if "<" in message or ">" in message:
        reduced = pangostrip(message)
        if ("<" in reduced or ">" in reduced) and not rgb :
            if message == reduced: # No pango markup
                message = message.replace("<", "&lt;").replace(">", "&gt;")
            else:
                print('"%s", line %d: < or > in pango string requires manual fix.' % (filename, line))
    return message

class WmllintIterator(WmlIterator):
    "Fold an Emacs-compatible error reporter into WmlIterator."
    def printError(self, *misc):
        """Emit an error locator compatible with Emacs compilation mode."""
        if not hasattr(self, 'lineno') or self.lineno == -1:
            print('"%s":' % self.fname, file=sys.stderr)
        else:
            print('"%s", line %d:' % (self.fname, self.lineno+1), end=" ", file=sys.stderr)
        for item in misc:
            print(item, end=" ", file=sys.stderr)
        print("", file=sys.stderr) #terminate line

def local_sanity_check(filename, nav, key, prefix, value, comment):
    "Sanity checks that don't require file context or globals."
    errlead = '"%s", line %d: ' %  (filename, nav.lineno+1)
    ancestors = nav.ancestors()
    in_definition = "#define" in ancestors
    in_call = [x for x in ancestors if x.startswith("{")]
    ignored = "wmllint: ignore" in nav.text
    parent = None
    if ancestors:
        parent = ancestors[-1]
        ancestors = ancestors[:-1]
    # Magic comment for adding custom conditional tags
    # Placed here rather than in global_sanity_check(), because
    # local_sanity_check() is used first
    # Do not use square braces while listing the new tags
    # they'll be automatically added by the code below
    # this is done to prevent possible interactions with other parts of the code
    m = re.search("# *wmllint: conditional tag +(.*)", nav.text)
    if m:
        for new_conditional in m.group(1).split(","):
            custom_conditionals.append("[" + new_conditional.strip() + "]")
    # Check for things marked translated that aren't strings or name generators
    if "_" in nav.text and not ignored:
        m = re.search(r'[=(]\s*_\s+("|<<)?', nav.text)
        if m and not m.group(1):
            print(errlead + 'translatability mark before non-string')
    # Most tags are not allowed with [part]
    if ("[part]" in ancestors or parent == "[part]") and isOpener(nav.element):
        # FIXME: this should be part of wmliterator's functionality
        if isExtender(nav.element):
            actual_tag = "[" + nav.element[2:]
        else:
            actual_tag = nav.element
        if actual_tag not in ("[part]", "[background_layer]", "[image]", "[insert_tag]",
                              "[if]", "[then]", "[elseif]", "[else]", "[switch]", "[case]",
                              "[variable]", "[deprecated_message]", "[wml_message]", "[have_unit]"):
            print(errlead + '%s not permitted within [part] tag' % nav.element)
    # Most tags are not permitted inside [if]
    if (len(ancestors) >= 1 and parent == "[if]") or \
       (len(ancestors) >= 2 and parent == "#ifdef" and ancestors[-1] == "[if]"):
        if isOpener(nav.element) and nav.element not in ("[and]",
                           "[else]", "[elseif]", "[frame]", "[have_location]",
                           "[have_unit]", "[not]", "[or]", "[then]", "[lua]",
                           "[variable]", "[true]", "[false]", "[found_item]",
                           "[proceed_to_next_scenario]", "[has_achievement]") \
                           and not nav.element.endswith("_frame]") \
                           and not nav.element.startswith("[filter") and \
                           nav.element not in custom_conditionals:
            print(errlead + 'illegal child of [if]:', nav.element)
    # Check for various things that shouldn't be outside an [ai] tag
    if not in_definition and not in_call and "[ai]" not in nav.ancestors() and "[+ai]" not in nav.ancestors() and not ignored:
        if key in ("number_of_possible_recruits_to_force_recruit",
                   "recruitment_ignore_bad_movement",
                   "recruitment_ignore_bad_combat",
                   "recruitment_pattern",
                   "villages_per_scout", "leader_value", "village_value",
                   "aggression", "caution", "attack_depth", "grouping", "advancements"):
            print(errlead + key + " outside [ai] scope")
    # Bad [recruit] attribute
    if parent in ("[allow_recruit]", "[disallow_recruit]") and key == "recruit":
        print(errlead + "recruit= should be type=")
    # Check [binary_path] and [textdomain] paths
    if parent == '[textdomain]' and key == 'path' and '/translations' not in value:
        print(errlead + 'no reference to "/translations" directory in textdomain path')
    if parent == '[binary_path]' and key == 'path':
        if '/external' in value or '/public' in value:
            print(errlead + '"/external" or "/public" image directories should no longer be used')
    # Accumulate data to check for missing next scenarios
    if parent == '[campaign]':
        if key == "first_scenario" and value != "null":
            nextrefs.append((filename, nav.lineno, value, nav.text))
    if parent == '[scenario]' or parent == None:
        if key == "next_scenario" and value != "null":
            nextrefs.append((filename, nav.lineno, value, nav.text))
        if key == 'id':
            scenario_to_filename[value] = filename
    # Check for campaign rank and warn if an unconventional value is found
    if parent == "[campaign]":
        if key == "rank" and not ignored:
            try:
                campaign_rank = int(value)
                # assume that UMCs are into the data/add-ons directory and mainline is into data/campaigns   
                if "add-ons" in filename:
                    if campaign_rank < 300:
                        print(errlead + "rank for add-on campaigns should be greater than or equal to 300")
                elif "campaigns" in filename:
                    if campaign_rank >= 300:
                        print(errlead + "rank for mainline campaigns should be between 0 and 299 (included)")  
            except ValueError:
                print(errlead + "rank must be a numeric value")

def global_sanity_check_events(filename, lines):
    """Part of global_sanity_check which finds each [event] tag.

    To handle nested events, this is an multi-pass implementation. This
    function will make one complete pass itself. Each subfunction called is
    given its own copy of the iterator, to do its own pass on the contents of
    the event without changing the iterator for this function.
    """
    deathcheck = True
    for nav in WmllintIterator(lines, filename):
        if "wmllint: deathcheck off" in nav.text:
            deathcheck = False
        elif "wmllint: deathcheck on" in nav.text:
            deathcheck = True
        # Now the tests. These run even when the opening tag is on a line which
        # turns that specific check off, in case the check is re-enabled before
        # the closing tag.
        if "[event]" in nav.text:
            sanity_check_speaks_in_death_event(nav.copy(), deathcheck)

def sanity_check_speaks_in_death_event(opening_tag, deathcheck):
    """Detect units that speak in their death events

    Given an iterator pointing to an [event]'s opening tag, check whether it's
    a die event, and if so check whether the already-dead unit speaks during
    it.

    This will move the iterator that is passed into it, the caller should make
    a copy if required.

    opening_tag -- an iterator on the [event] tag's line
    deathcheck -- the status of the global "deathcheck on/off" flag at
    the start of this event
    """
    filter_subject = None
    die_event = False
    if not opening_tag.hasNext():
        # Either this is an empty tag, or the closing tag must be
        # missing. There will be a separate error reported for that.
        return
    opening_tag.__next__()
    event_scope = opening_tag.iterScope()
    base_depth = len(event_scope.ancestors())

    # First pass - find out if the event is a death event, and if it has a
    # filter. Here the base_depth is used to ignore nested events and other
    # tags that have a [filter] subtag.
    for nav in event_scope.copy():
        ancestors = nav.ancestors()
        if len(ancestors) == base_depth + 1:
            fields = parse_attribute(nav.text)
            if fields:
                (key, prefix, value, comment) = fields
                if key == 'name' and value == 'die':
                    die_event = True
        if len(ancestors) == base_depth + 2:
            if ancestors[-1] == "[filter]":
                fields = parse_attribute(nav.text)
                if fields:
                    (key, prefix, value, comment) = fields
                    if key == 'id':
                        filter_subject = value

    # Second pass - check if the subject speaks in this event. This will
    # descend in to nested events, but as the unit is already dead it shouldn't
    # speak in those either.
    if die_event and filter_subject:
        for nav in event_scope:
            if "wmllint: deathcheck off" in nav.text:
                deathcheck = False
                continue
            elif "wmllint: deathcheck on" in nav.text:
                deathcheck = True
            parent = nav.ancestors()[-1]
            if parent == "[message]":
                # Who is speaking?
                fields = parse_attribute(nav.text)
                if fields:
                    (key, prefix, value, comment) = fields
                    if key in ("id", "speaker"):
                        if deathcheck and ((value == filter_subject) or (value == "unit")):
                            print('"%s", line %d: %s speaks in his/her "die" event rather than "last breath"' \
                                  % (nav.fname, nav.lineno+1, value))

def global_sanity_check(filename, lines):
    "Perform sanity and consistency checks on input files."
    # Build dictionaries of unit movement types and races
    in_unit_type = None
    unit_id = ""
    base_unit = ""
    for nav in WmllintIterator(lines, filename):
        if nav.element == "[unit_type]":
            unit_race = ""
            unit_id = ""
            unit_level = None
            base_unit = ""
            in_unit_type = nav.lineno + 1
            hitpoints_specified = False
            unit_usage = ""
            temp_movetypes = []
            temp_races = []
            temp_advances = []
            continue
        elif nav.element == "[/unit_type]":
            if unit_id and unit_usage:
                usage[unit_id] = unit_usage
            if unit_id and (unit_level is not None):
                unit_levels[unit_id] = unit_level
            if unit_id and temp_movetypes:
                for movetype in temp_movetypes:
                    # movetype, race and advance are 3-element tuples, expand them
                    unit_movetypes.append((unit_id, *movetype))
            if unit_id and temp_races:
                for race in temp_races:
                    unit_races.append((unit_id, *race))
            if unit_id and temp_advances:
                for advance in temp_advances:
                    advances.append((unit_id, *advance)) 
            if unit_id and base_unit:
                derived_units.append((filename, nav.lineno + 1, unit_id, base_unit))
            if unit_id and not base_unit:
                # If the unit didn't specify hitpoints, there is some wacky
                # stuff going on (possibly pseudo-[base_unit] behavior via
                # macro generation) so disable some of the consistency checks.
                if not hitpoints_specified:
                    continue
                if "[theme]" not in nav.ancestors() and "[base_unit]" not in nav.ancestors() and not unit_race:
                    print('"%s", line %d: unit %s has no race' \
                         % (filename, in_unit_type, unit_id))
            in_unit_type = None
            unit_id = ""
            unit_level = None
            base_unit = ""
            unit_race = None
            unit_usage = ""
            temp_movetypes = []
            temp_races = []
            temp_advances = []
        # the glob pattern matches any WML tag starting with filter, including [filter] itself
        if '[unit_type]' in nav.ancestors() and not nav.glob_ancestors("[[]filter*[]]"):
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                if key == "id":
                    if value[0] == "_":
                        value = value[1:].strip()
                    if not unit_id and "[base_unit]" not in nav.ancestors():
                        unit_id = value
                        unit_types.append(unit_id)
                    if not base_unit and "[base_unit]" in nav.ancestors():
                        base_unit = value
                elif key == "hitpoints":
                    hitpoints_specified = True
                elif key == "level":
                    # levels are signed integers, however here we leave them as strings
                    # because we need to check for them in recruitment patterns (consistency_check())
                    unit_level = value
                elif key == "usage":
                    unit_usage = value
                elif key == "movement_type":
                    if '{' not in value:
                        temp_movetypes.append((filename, nav.lineno + 1, value))
                elif key == "race":
                    if '{' not in value:
                        unit_race = value
                        temp_races.append((filename, nav.lineno + 1, unit_race))
                elif key == "advances_to":
                    advancements = value
                    if advancements.strip() != "null":
                        temp_advances.append((filename, nav.lineno + 1, advancements))
            except TypeError:
                pass

    # Sanity-check all the [event] tags
    global_sanity_check_events(filename, lines)

    # Collect information on defined movement types and races
    for nav in WmllintIterator(lines, filename):
        above = nav.ancestors()
        if above and above[-1] in ("[movetype]", "[race]"):
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                if above[-1] == "[movetype]" and key == 'name':
                    movetypes.append(value)
                if above[-1] == "[race]" and key == 'id':
                    races.append(value)
            except TypeError:
                pass
    # Sanity-check recruit and recruitment_pattern.
    # This code has a limitation; if there are multiple instances of
    # recruit and recruitment_pattern (as can happen if these lists
    # vary by EASY/NORMAL/HARD level) this code will only record the
    # last of each for later consistency checking.
    in_side = False
    recruit = {}
    recruitment_pattern = {}
    ifdef_stack = [None]
    for num, line in enumerate((l.strip() for l in lines), start=1):
        if line.startswith("#ifdef") or line.startswith("#ifhave") or line.startswith("#ifver"):
            ifdef_stack.append(line.split()[1])
            continue
        if line.startswith("#ifndef") or line.startswith("#ifnhave") or line.startswith("#ifnver"):
            ifdef_stack.append("!" + line.split()[1])
            continue
        if line.startswith("#else"):
            if ifdef_stack[-1].startswith("!"):
                ifdef_stack.append(ifdef_stack[-1][1:])
            else:
                ifdef_stack.append("!" + ifdef_stack[-1])
            continue
        if line.startswith("#endif"):
            ifdef_stack.pop()
            continue
        precomment = line.split("#")[0]
        if "[side]" in precomment:
            in_side = True
            continue
        elif "[/side]" in precomment:
            if recruit or recruitment_pattern:
                sides.append((filename, recruit, recruitment_pattern))
            in_side = False
            recruit = {}
            recruitment_pattern = {}
            continue
        if not in_side or '=' not in precomment:
            continue
    # Sanity check ellipses
    # Starting from 1.11.5, units with canrecruit=yes gain automatically a leader ellipse
    # Starting from 1.11.6, units without a ZoC gain automatically a nozoc ellipse
    # Check if ellipse= was used and warn if so
    # Do not warn if misc/ellipse-hero was used, since it isn't automatically assigned by C++
    # and it's assigned/removed with IS_HERO/MAKE_HERO/UNMAKE_HERO
    # magic comment wmllint: no ellipsecheck deactivates this check for the current line
    in_effect = False
    in_unit = False
    in_side = False
    in_unit_type = False
    for num, line in enumerate(lines, start=1):
        if has_opening_tag(line, "effect"):
            in_effect = True
        elif "[/effect]" in line:
            in_effect = False
        elif has_opening_tag(line, "unit"):
            in_unit = True
        elif "[/unit]" in line:
            in_unit = False
        elif has_opening_tag(line, "side"):
            in_side = True
        elif "[/side]" in line:
            in_side = False
        elif has_opening_tag(line, "unit_type"):
            in_unit_type = True
        elif "[/unit_type]" in line:
            in_unit_type = False
        # ellipsecheck magic comment allows to deactivate the ellipse sanity check
        if "wmllint: no ellipsecheck" not in line:
            if in_effect:
                try:
                    (key, prefix, value, comment) = parse_attribute(line)
                    if key == "ellipse" and value in ("misc/ellipse-nozoc", "misc/ellipse-leader"):
                        print('"%s", line %d: [effect] apply_to=ellipse needs to be removed' % (filename, num))
                    elif key == "ellipse" and value not in ("none", "misc/ellipse", "misc/ellipse-hero"):
                        print('"%s", line %d: custom ellipse %s may need to be updated' % (filename, num, value))
                except TypeError: # this is needed to handle tags, that parse_attribute cannot split
                    pass
            elif in_unit or in_side or in_unit_type:
                try:
                    (key, prefix, value, comment) = parse_attribute(line)
                    if key == "ellipse" and value in ("misc/ellipse-nozoc","misc/ellipse-leader"):
                        print('"%s", line %d: %s=%s needs to be removed' % (filename, num, key, value))
                    elif key == "ellipse" and value not in ("none","misc/ellipse","misc/ellipse-hero"):
                        print('"%s", line %d: custom ellipse %s may need to be updated' % (filename, num, value))
                except TypeError: # this is needed to handle tags, that parse_attribute cannot split
                    pass
    # Handle [advancefrom] deprecation
    # it should be replaced with [modify_unit_type]
    # but, unlike the former tag, the new one goes in the _main.cfg, even as a macro call
    # Because we can't be sure about what the UMC author wants to do, just warn
    in_campaign = False
    for num, line in enumerate(lines, start=1):
        precomment = line.split("#")[0]
        if "[campaign]" in precomment:
            in_campaign = True
            continue
        if "[/campaign]" in precomment:
            in_campaign = False
            continue
        if has_opening_tag(precomment, "advancefrom"):
            print("{}, line {}: [advancefrom] needs to be manually updated to \
[modify_unit_type] and moved into the _main.cfg file".format(filename, num))
        if in_campaign:
            try:
                (key, prefix, value, comment) = parse_attribute(line)
                if key == "extra_defines":
                    print("{}, line {}: extra_defines are now macros and need \
to be called on their own".format(filename, num))
            except TypeError:
                pass
    # Interpret various magic comments
    for line in lines:
        # Interpret magic comments for setting the usage pattern of units.
        # This coped with some wacky UtBS units that were defined with
        # variant-spawning macros.  The prototype comment looks like this:
        #wmllint: usage of "Desert Fighter" is fighter
        m = re.search('# *wmllint: usage of "([^"]*)" is +(.*)', line)
        if m:
            usage[m.group(1)] = m.group(2).strip()
            unit_types.append(m.group(1))
        # Magic comment for adding non-standard usage types
        m = re.search('# *wmllint: usagetypes? +(.*)', line)
        if m:
            for newusage in m.group(1).split(","):
                standard_usage_types.add(newusage.strip())
    # Consistency-check the id= attributes in [side], [unit], [recall],
    # and [message] scopes, also correctness-check translation marks and look
    # for double spaces at end of sentence.
    extra_translatables = []
    in_scenario = False
    in_multiplayer = False
    subtag_depth = 0
    in_message = False
    in_option = False
    #last index is true: we're currently directly in an [event]
    #this avoids complaints about unknown [event]id=something, but keeps the check
    #in case some [filter]id= comes in this [event]
    directly_in_event = []
    in_store = False
    in_not = False
    in_name_generator = False
    in_lua = False
    in_lua_args = False
    in_window = False
    in_resolution = False
    in_gui_definition = False
    in_deprecated_message = False
    in_chat = False
    storeid = None
    ignoreable = False
    sentence_end = re.compile("(?<=[.!?;:])  +")
    capitalization_error = re.compile("(?<=[.!?])  +[a-z]")
    markcheck = True
    translation_mark = re.compile(r'_ *"')
    name_generator_re = re.compile(r"name_generator\s*=\s*_\s*<<")
    gui_def_regex = re.compile(r"\[/.*\_definition\]")
    for i in range(len(lines)):
        for (item, warning) in warn_if_found:
            if item in lines[i]:
                if "#" not in lines[i] or ("#" in lines[i] and lines[i].find("#") > lines[i].find(item)):
                    print(item +": "+ warning)
        extras = re.search("# *wmllint: translatables +(.*)", lines[i])
        if extras:
            extra_translatables.append(string_strip(extras.group(1)).strip())
        if has_opening_tag(lines[i], "lua"):
            in_lua = True
        elif "[/lua]" in lines[i]:
            in_lua = False
        if in_lua:
            if has_opening_tag(lines[i], "args"):
                in_lua_args = True
            elif "[/args]" in lines[i]:
                in_lua_args = False
            if not in_lua_args:
                # As far as I can tell none of the below tests make sense inside of a lua tag
                continue
        # This logic looks odd because a scenario can be conditionally
        # wrapped in both [scenario] and [multiplayer]; we mustn't count
        # either as a subtag even if it occurs inside the other, otherwise
        # this code might see id= declarations as being at the wrong depth.
        if "[scenario]" in lines[i]:
            in_scenario = True
        elif "[/scenario]" in lines[i]:
            in_scenario = False
        elif "[multiplayer]" in lines[i]:
            in_multiplayer = True
        elif "[/multiplayer]" in lines[i]:
            in_multiplayer = False
        else:
            # the + check is needed for augmentation tags
            # otherwise subtag_depth may become negative
            if re.search(r"\[\+?[a-z]", lines[i]):
                subtag_depth += 1
            if "[/" in lines[i]:
                subtag_depth -= 1
        if has_opening_tag(lines[i], "event"):
            directly_in_event.append(True)
        elif re.search(r"\[\+?[a-z]", lines[i]):
            directly_in_event.append(False)
        elif "[/" in lines[i]:
            if len(directly_in_event) > 0:
                directly_in_event.pop()
        # Ordinary subtag flags begin here
        if has_opening_tag(lines[i], "message"):
            in_message = True
        elif "[/message]" in lines[i]:
            in_message = False
        elif has_opening_tag(lines[i], "option"):
            in_option = True
        elif "[/option]" in lines[i]:
            in_option = False
        elif has_opening_tag(lines[i], "kill") or \
             has_opening_tag(lines[i], "effect") or \
             has_opening_tag(lines[i], "move_unit_fake") or \
             has_opening_tag(lines[i], "scroll_to_unit"):
            ignoreable = True
        elif "[/kill]" in lines[i] or "[/effect]" in lines[i] or "[/move_unit_fake]" in lines[i] or "[/scroll_to_unit]" in lines[i]:
            ignoreable = False
        elif has_opening_tag(lines[i], "store_unit"):
            in_store = True
        elif "[/store_unit]" in lines[i]:
            in_store = False
            storeid = None
        elif has_opening_tag(lines[i], "not"):
            in_not = True
        elif "[/not]" in lines[i]:
            in_not = False
        elif has_opening_tag(lines[i], "window"):
            in_window = True
        elif "[/window]" in lines[i]:
            in_window = False
        elif has_opening_tag(lines[i], "resolution"):
            in_resolution = True
        elif "[/resolution]" in lines[i]:
            in_resolution = False
        elif "_definition]" in lines[i] and "[/" not in lines[i]:
            in_gui_definition = True
        elif gui_def_regex.search(lines[i]):
            in_gui_definition = False
        elif has_opening_tag(lines[i], "deprecated_message"):
            in_deprecated_message = True
        elif "[/deprecated_message]" in lines[i]:
            in_deprecated_message = False
        elif has_opening_tag(lines[i], "chat"):
            in_chat = True
        elif "[/chat]" in lines[i]:
            in_chat = False
        elif name_generator_re.search(lines[i]):
            in_name_generator = True
        elif in_name_generator and ">>" in lines[i]:
            in_name_generator = False
        if "wmllint: markcheck off" in lines[i]:
            markcheck = False
        elif "wmllint: markcheck on" in lines[i]:
            markcheck = True
        # Parse recruit/recall macros to recognize characters.  This section
        # assumes that such a macro is the first item on a line.
        leadmac = re.match(r'{[^}\s]+.', lines[i].lstrip())
        if leadmac:
            macname = leadmac.group()[1:-1]
            if not leadmac.group().endswith('}'):
                # Update 1.4's {LOYAL_UNIT} macro to {NAMED_LOYAL_UNIT}.
                if macname == 'LOYAL_UNIT':
                    (args, optional_args, is_unfinished) = parse_macroref(0, leadmac.string)
                    if len(args) == 6:
                        lines[i] = lines[i].replace('{LOYAL_UNIT', '{NAMED_LOYAL_UNIT', 1)
        m = re.search("# *wmllint: recognize +(.*)", lines[i])
        if '=' not in lines[i] or ignoreable:
            continue
        parseable = False
        try:
            (key, prefix, value, comment) = parse_attribute(lines[i])
            parseable = True
        except TypeError:
            pass
        if parseable:
            if "wmllint: ignore" in comment:
                continue
            # Recognize units when unstored
            if (in_scenario or in_multiplayer) and in_store:
                if key == 'id' and not in_not:
                    if not storeid == None:
                        storeid == storeid + ',' + value
                    else:
                        storeid = value
            has_tr_mark = translation_mark.search(value)
            if has_tr_mark:
                # FIXME: This test is rather bogus as is.
                # Doing a better job would require tokenizing to pick up the
                # string boundaries.
                if '{' in value and "+" not in value and value.find('{') > value.find("_"):
                    print('"%s", line %d: macro reference in translatable string'\
                          % (filename, i+1))
            # Check correctness of translation marks and descriptions
            if key.startswith("{") or key.startswith("<") or key.startswith("("):
                pass
            elif in_name_generator:
                pass
            elif key in ('name', 'male_name', 'female_name', 'value', 'info', 'config_name', 'deserters_names'): # FIXME: check this someday
                pass
            elif key == "variation_name":
                if markcheck and not has_tr_mark:
                    print('"%s", line %d: %s should be renamed as variation_id and/or marked as translatable' \
                          % (filename, i+1, key))
            elif is_translatable(key, extra_translatables):
                # if there's a value, translation mark checking is enabled, and there's no translation mark
                # and there are no other magic comments disbling the check
                # and it's not the first line of a concatenated string which has only pango markup
                # and it's not the description attribute in [window]
                # and it's not the description attribute in a gui definition like [button_definition] or [menu_button_definition]
                # and it's not the message attribute in [chat], [deprecated_message]
                if value and markcheck and not has_tr_mark \
                and not value.startswith("$") and not value.startswith("{") \
                and not ("wmllint: ignore" in comment or "wmllint: noconvert" in comment) \
                and not (value.startswith('<') and value.endswith('+')) \
                and (not in_window or (in_window and in_resolution)) \
                and not in_gui_definition \
                and not in_chat and not in_deprecated_message:
                    # [image] sometimes uses label to specify a reference to a PNG or WebP file
                    # so do not mark for translation in those cases
                    # and ignore 'text = "(text)"' used in [text]
                    if not (key == 'label' and ('.png' in value or '.webp' in value)) and not value == '(text)' and not value == '(revision_number)':
                        # ignore attributes where the translatable string is on the next line in order to have a po comment
                        if len(lines) >= i+1 and not (lines[i+1].strip().startswith('_') or lines[i+1].strip().startswith('#po')):
                            print('"%s", line %d: %s needs translation mark' % (filename, i+1, key))
                            lines[i] = lines[i].replace('=', "=_ ", 1)
                nv = sentence_end.sub(" ", value)
                if nv != value:
                    print('"%s", line %d: double space after sentence end' \
                          % (filename, i+1))
                    lines[i] = sentence_end.sub(" ", lines[i])
                if capitalization_error.search(lines[i]):
                    print('"%s", line %d: probable capitalization or punctuation error' \
                          % (filename, i+1))
                if key == "message" and in_message and not in_option and not ("wmllint: ignore" in comment or "wmllint: noconvert" in comment):
                    lines[i] = pangoize(lines[i], filename, i)
            else:
                if markcheck and has_tr_mark and not ("wmllint: ignore" in comment or "wmllint: noconvert" in comment):
                    print('"%s", line %d: %s should not have a translation mark' \
                          % (filename, i+1, key))
                    lines[i] = prefix + value.replace("_", "", 1) + comment + '\n'
    # Check for textdomain strings; should be exactly one, on line 1
    textdomains = []
    no_text = False
    for num, line in enumerate(lines, start=1):
        if "#textdomain" in line:
            textdomains.append(num)
        elif "wmllint: no translatables" in line:
            no_text = True
    if not no_text:
        if not textdomains:
            print('"%s", line 1: no textdomain string' % filename)
        elif textdomains[0] == 1: # Multiples are OK if first is on line 1
            pass
        elif len(textdomains) > 1:
            print('"%s", line %d: multiple textdomain strings on lines %s' % \
                  (filename, textdomains[0], ", ".join(map(str, textdomains))))
        else:
            w = textdomains[0]
            print('"%s", line %d: single textdomain declaration not on line 1.' % \
                  (filename, w))
            lines = [lines[w-1].lstrip()] + lines[:w-1] + lines[w:]
    return lines

def condition_match(p, q):
    "Do two condition-states match?"
    # The empty condition state is represented by None
    if p is None or q is None or (p == q):
        return True
    # Past this point it's all about handling cases with negation
    sp = p
    np = False
    if sp.startswith("!"):
        sp = sp[1:]
        np = True
    sq = q
    nq = False
    if sq.startswith("!"):
        sq = sp[1:]
        nq == True
    return (sp != sq) and (np != nq)

def consistency_check():
    "Consistency-check state information picked up by sanity_check"
    derivations = {u[2]: u[3] for u in derived_units}
    for (filename, recruitdict, patterndict) in sides:
        for (rdifficulty, (rl, recruit)) in recruitdict.items():
            # using sets to automatically remove duplicates
            usage_types = set()
            available_levels = set()
            for rtype in recruit:
                base = rtype
                if rtype not in unit_types:
                    # Assume WML coder knew what he was doing if macro reference
                    if not rtype.startswith("{"):
                        print('"%s", line %d: %s is not a known unit type' % (filename, rl, rtype))
                    continue
                elif rtype not in usage:
                    if rtype in derivations:
                        base = derivations[rtype]
                    else:
                        print('"%s", line %d: %s has no usage type' % \
                              (filename, rl, rtype))
                        continue
                if not base in usage:
                    print('"%s", line %d: %s has unknown base %s' % \
                          (filename, rl, rtype, base))
                    continue
                else:
                    utype = usage[base]
                usage_types.add(utype)
                # collect available unit levels for recruitment
                if rtype in unit_levels:
                    available_levels.add(unit_levels[rtype])
                elif rtype in derivations:
                    base = derivations[rtype]
                    available_levels.add(unit_levels[base])
                else:
                    print('"{}", line {}: {} has no level'.format(filename, rl, rtype))
                for (pdifficulty, (pl, recruit_pattern)) in patterndict.items():
                    if condition_match(pdifficulty, rdifficulty):
                        # now recruitment patterns can also contain levels and unit types, not just usages
                        # so check all these three options
                        if (utype not in recruit_pattern) and \
                           (unit_levels[rtype] not in recruit_pattern) and \
                           (rtype not in recruit_pattern):
                            rshow = ''
                            if rdifficulty is not None:
                                rshow = 'At ' + rdifficulty + ', '
                            ushow = ''
                            if utype not in standard_usage_types:
                                ushow = ', a possible non-standard usage class'
                            pshow = ''
                            if pdifficulty is not None:
                                pshow = ' ' + pdifficulty
                            print('"%s", line %d: %s%s (%s%s) doesn\'t match the%s recruitment pattern (%s) for its side' \
                                  % (filename, rl, rshow, rtype, utype, ushow, pshow, ", ".join(recruit_pattern)))
            # We have a list of all the usage types recruited at this difficulty
            # in usage_types. Use it to check the matching pattern, if any. Suppress
            # this check if the recruit line is a macroexpansion.
            if recruit and not recruit[0].startswith("{"):
                for (pdifficulty, (pl, recruitment_pattern)) in patterndict.items():
                    if condition_match(pdifficulty, rdifficulty):
                        for utype in recruitment_pattern:
                            # again, check for levels and unit types as well
                            if (utype not in usage_types) and \
                               (utype not in available_levels) and \
                               (utype not in recruit):
                                rshow = '.'
                                if rdifficulty is not None:
                                    rshow = ' at difficulty ' + rdifficulty + '.'
                                ushow = ''
                                # unit types usually have at least one uppercase character
                                # levels have only numeric characters
                                # so assume that a completely lowercase string is a usage type
                                if utype not in standard_usage_types and utype.islower():
                                    ushow = ' (a possible non-standard usage class)'
                                elif utype.isnumeric():
                                    ushow = ' (a possible unit level)'
                                print('"%s", line %d: no %s%s units recruitable%s' % (filename, pl, utype, ushow, rshow))
    if movetypes:
        for (unit_id, filename, line, movetype) in unit_movetypes:
            if movetype not in movetypes:
                print('"%s", line %d: %s has unknown movement type' \
                      % (filename, line, unit_id))
    if races:
        for (unit_id, filename, line, race) in unit_races:
            if race not in races:
                print('"%s", line %d: %s has unknown race' \
                      % (filename, line, unit_id))
    # Should we be checking the transitive closure of derivation?
    # It's not clear whether [base_unit] works when the base is itself derived.
    for (filename, line, unit_type, base_unit) in derived_units:
        # ignore when the base unit contains a macro argument
        if base_unit not in unit_types and "{" not in base_unit:
            print('"%s", line %d: derivation of %s from %s does not resolve' \
                  % (filename, line, unit_type, base_unit))
    # Check that all advancements are known units
    for (unit_id, filename, lineno, advancements) in advances:
        advancements = [elem.strip() for elem in advancements.split(",")]
        known_units = unit_types + list(derivations.keys())
        # also skip cases where a macro argument is part of the advancement
        bad_advancements = [x for x in advancements if x not in known_units and "{" not in x]
        if bad_advancements:
            print('"%s", line %d: %s has unknown advancements %s' \
                  % (filename, lineno, unit_id, bad_advancements))
    # Check next-scenario pointers
    for (filename, lineno, value, line) in nextrefs:
        if value not in scenario_to_filename and "no-syntax-rewrite" not in line:
            print('"%s", line %d: unresolved scenario reference %s' % \
                  (filename, lineno, value))

# Syntax transformations

leading_ws = re.compile(r"^\s*")

def leader(s):
    "Return a copy of the leading whitespace in the argument."
    return leading_ws.match(s).group(0)

def hack_syntax(filename, lines):
    # Syntax transformations go here.  This gets called once per WML file;
    # the name of the file is passed as filename, text of the file as the
    # array of strings in lines.  Modify lines in place as needed;
    # changes will be detected by the caller.
    #
    # Deal with a few Windows-specific problems for the sake of cross-
    # platform harmony. First, the use of backslashes in file paths.
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        # Looking out for "#" used for color markup
        precomment = re.split(r'\s#', lines[i], 1)[0]
        comment = lines[i][len(precomment):]
        if '\\' in precomment:
            while re.search(r'(?<!\\)\\(?!\\)[^ ={}"]+\.(png|ogg|wav|webp|jpe?g|map|mask|cfg)\b', precomment, flags=re.IGNORECASE):
                backslash = re.search(r'([^ ={}"]*(?<!\\)\\(?!\\)[^ ={}"]+\.)(png|ogg|wav|webp|jpe?g|map|mask|cfg)(?=\b)', precomment, flags=re.IGNORECASE)
                fronted = backslash.group(1).replace("\\","/") + backslash.group(2)
                precomment = precomment[:backslash.start()] + fronted + precomment[backslash.end():]
                print('"%s", line %d: %s -> %s -- please use frontslash (/) for cross-platform compatibility' \
                      % (filename, i+1, backslash.group(), fronted))
        # Then get rid of the 'userdata/' headache.
        if 'userdata/' in precomment:
            while re.search(r'user(data/)?data/[ac]', precomment):
                userdata = re.search(r'(?:\.\./)?user(?:data/)?(data/[ac][^/]*/?)', precomment)
                precomment = precomment[:userdata.start()] + userdata.group(1) + precomment[userdata.end():]
                print('"%s", line %d: %s -> %s -- DO NOT PREFIX PATHS WITH "userdata/"' \
                      % (filename, i+1, userdata.group(), userdata.group(1)))
        lines[i] = precomment + comment

    # Ensure that every attack has a translatable description.
    in_filter_wml = False
    attack_left_open = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        elif "[filter_wml]" in lines[i]:
            in_filter_wml = True
        elif "[/filter_wml]" in lines[i]:
            in_filter_wml = False
        elif "[attack]" in lines[i]:
            if "#" in lines[i] and lines[i].find("#") < lines[i].find("[attack]"):
                continue
            j = i
            have_description = False
            while '[/attack]' not in lines[j]:
                if lines[j].strip().startswith("description"):
                    have_description = True
                j += 1
                # unterminated [attack] tags, for example embedded in a macro,
                # lead to a crash. Print an error message, skip this loop
                # and the following one
                if j >= len(lines):
                    print("{}, line {}: [attack] tag not closed in this file".format(filename, i+1))
                    attack_left_open = True
                    break
            if (not have_description) and (not attack_left_open):
                j = i
                while '[/attack]' not in lines[j]:
                    fields = lines[j].strip().split('#')
                    syntactic = fields[0]
                    comment = ""
                    if len(fields) > 1:
                        comment = fields[1]
                    if syntactic.strip().startswith("name"):
                        description = syntactic.split("=")[1].strip()
                        if not description.startswith('"'):
                            description = '"' + description + '"\n'
                        # Skip the insertion if this is a dummy declaration
                        # or one modifying an attack inherited from a base unit.
                        if "no-icon" not in comment and not in_filter_wml:
                            new_line = leader(syntactic) + "description=_"+description
                            if verbose:
                                print('"%s", line %d: inserting %s' % (filename, i+1, repr(new_line)))
                            lines.insert(j+1, new_line)
                            j += 1
                    j += 1
    # Hack tracking-map macros from 1.4 and earlier.  The idea is to lose
    # all assumptions about colors in the names
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        elif "{DOT_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("DOT_CENTERED", "NEW_JOURNEY")
        elif "{DOT_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("DOT_WHITE_CENTERED", "OLD_JOURNEY")
        elif "{CROSS_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("CROSS_CENTERED", "NEW_BATTLE")
        elif "{CROSS_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("CROSS_WHITE_CENTERED", "OLD_BATTLE")
        elif "{FLAG_RED_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("FLAG_RED_CENTERED", "NEW_REST")
        elif "{FLAG_WHITE_CENTERED" in lines[i]:
            lines[i] = lines[i].replace("FLAG_WHITE_CENTERED", "OLD_REST")
        elif "{DOT " in lines[i] or "CROSS" in lines[i]:
            m = re.search("{(DOT|CROSS) ([0-9]+) ([0-9]+)}", lines[i])
            if m:
                n = m.group(1)
                if n == "DOT":
                    n = "NEW_JOURNEY"
                if n == "CROSS":
                    n = "NEW_BATTLE"
                x = int(m.group(2)) + 5
                y = int(m.group(3)) + 5
                lines[i] = lines[i][:m.start(0)] +("{%s %d %d}" % (n, x, y)) + lines[i][m.end(0):]
    # Palette transformation for 1.7:
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        # RC -> PAL
        elif "RC" in lines[i]:
            lines[i] = re.sub(r"~RC\(([^=\)]*)=([^)]*)\)",r"~PAL(\1>\2)",lines[i])
    # Rename the terrain definition tag
    in_standing_anim = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        # Ugh...relies on code having been wmlindented
        lines[i] = re.sub(r"^\[terrain\]", "[terrain_type]", lines[i])
        lines[i] = re.sub(r"^\[/terrain\]", "[/terrain_type]", lines[i])
        if has_opening_tag(lines[i], "standing_anim"):
            in_standing_anim = True
        if "[/standing_anim]" in lines[i]:
            in_standing_anim = False
        if in_standing_anim:
            lines[i] = re.sub(r"terrain([^_])", r"terrain_type\1", lines[i])
    # Rename two attributes in [set_variable]
    in_set_variable = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        if has_opening_tag(lines[i], "set_variable"):
            in_set_variable = True
        if "[/set_variable]" in lines[i]:
            in_set_variable = False
        if in_set_variable:
            lines[i] = re.sub(r"format(?=\s*=)", r"value", lines[i])
            lines[i] = re.sub(r"random(?=\s*=)", r"rand", lines[i])
    # campaigns directory becomes add-ons
    in_binary_path = in_textdomain = False
    for i in range(len(lines)):
        if "no-syntax-rewrite" in lines[i]:
            break
        if lines[i].lstrip().startswith("#"):
            pass
        # This is done on every line
        if "campaigns/" in lines[i]:
            lines[i] = lines[i].replace("{~campaigns/", "{~add-ons/")
            lines[i] = lines[i].replace("{~/campaigns/", "{~add-ons/")
            lines[i] = lines[i].replace("{@campaigns/", "{~add-ons/")
            # Convert UMC to data/add-ons without clobbering mainline. Each path
            # is checked against a list of mainline campaigns. UMC paths are
            # updated to "data/add-ons/"; mainline path strings are unaltered.
            x = 0
            for dc in re.finditer(r"data/campaigns/(\w[\w'&+-]*)", lines[i]):
                if dc.group(1) in mainline:
                    continue
                # Because start() and end() are based on the original position
                # of each iteration, while each replacement shortens the line
                # by two characters, we must subtract an increment that grows
                # with each substitution.
                lines[i] = lines[i][:dc.start()-x] + 'data/add-ons/' + dc.group(1) + lines[i][dc.end()-x:]
                x = x+2
                print('"%s", line %d: data/campaigns/%s -> data/add-ons/%s'\
                      %(filename, i+1, dc.group(1), dc.group(1)))
        elif "@add-ons/" in lines[i]:
            lines[i] = lines[i].replace("{@add-ons/", "{~add-ons/")
        # Occasionally authors try to use '~' with [textdomain] or [binary_path].
        if has_opening_tag(lines[i], "binary_path"):
            in_binary_path = True
        if "[/binary_path]" in lines[i]:
            in_binary_path = False
        if has_opening_tag(lines[i], "textdomain"):
            in_textdomain = True
        if "[/textdomain]" in lines[i]:
            in_textdomain = False
        if in_binary_path or in_textdomain:
            if '~' in lines[i]:
                tilde = re.search(r'(^\s*path) *= *([^#]{0,5})(~/?(data/)?add-ons/)', lines[i])
                if tilde:
                    lines[i] = tilde.group(1) + '=' + tilde.group(2) + 'data/add-ons/' + lines[i][tilde.end():]
                    print('"%s", line %d: %s -> data/add-ons/ -- [textdomain] and [binary_path] paths do not accept "~" for userdata'\
                          % (filename, i+1, tilde.group(3)))
    # some tags do no longer support default side=1 attribute but may use [filter_side]
    # with a SSF instead
    # (since 1.9.5, 1.9.6)
    if missingside:
        side_one_tags_allowing_filter_side = (
            ("remove_shroud"),
            ("place_shroud"),
            ("gold"),
            ("modify_side"),
            ("modify_ai")
            )
        outside_of_theme_wml = True # theme wml contains a [gold] tag - exclude that case
        in_side_one_tag = False
        side_one_tag_needs_side_one = True
        for i in range(len(lines)):
            if "no-syntax-rewrite" in lines[i]:
                break
            precomment = lines[i].split("#")[0]
            if outside_of_theme_wml:
                if has_opening_tag(precomment, "theme"):
                    outside_of_theme_wml = False
            else:
                if "[/theme]" in precomment:
                    outside_of_theme_wml = True
            if outside_of_theme_wml:
                if not in_side_one_tag:
                    for tag in side_one_tags_allowing_filter_side:
                        if "[" + tag + "]" in precomment:
                            in_side_one_tag = True
                else:
                    if side_one_tag_needs_side_one:
                        if "side=" in precomment:
                            side_one_tag_needs_side_one = False
                        if "[filter_side]" in precomment:
                            side_one_tag_needs_side_one = False
                    for tag in side_one_tags_allowing_filter_side:
                        if "[/" + tag + "]" in precomment:
                            if side_one_tag_needs_side_one:
                                if verbose:
                                    print('"%s", line %d: [%s] without "side" attribute is now applied to all sides'%(filename, i+1, tag))
                            in_side_one_tag = False
                            side_one_tag_needs_side_one = True
                            break
    # handle removal of obsolete lines without replacement
    # obsolete_lines contains normal strings, so re.escape is needed to embed them into this regex
    obsolete_re = re.compile(r"(\s*)(" + "|".join([re.escape(l) for l in obsolete_lines]) + r")(.*)")
    for i, line in enumerate(lines):
        if "no-syntax-rewrite" in line:
            break
        if "wmllint: noconvert" in line:
            continue
        m = obsolete_re.match(line)
        if m:
            # match group 1 is the indentation
            # match group 3 contains everything after the removed part (like comments)
            new_line = m.group(1) + m.group(3).lstrip()
            if new_line.isspace():
                # only empty spaces, make the line blank
                # don't remove the line to not alter the other for cycles
                lines[i] = "\n"
            else:
                # keep indentation and comments
                lines[i] = new_line + "\n"
            # match group 2 contains the removed part
            print('"{}", line {}: removed {}'.format(filename, i+1, m.group(2)))
    # handle deprecation of [unit] placement=map_overwrite/map_passable/leader_passable
    in_unit = False
    for i, line in enumerate(lines):
        if "no-syntax-rewrite" in line:
            break
        if has_opening_tag(line, "unit"):
            in_unit = True
        elif "[/unit]" in line:
            in_unit = False
        if in_unit:
            # once again, keep indentation and comments when replacing
            m = re.match(r"(\s*)placement\s?=\s?(map_overwrite|map_passable|leader_passable)(.*)", line)
            if m:
                if m.group(2) == "map_overwrite":
                    lines[i] = "{}placement=map{}\n{}overwrite=yes\n".format(m.group(1), m.group(3), m.group(1))
                    print('"{}", line {}: placement=map_overwrite -> placement=map overwrite=yes'.format(filename,
                                                                                                         i+1))
                elif m.group(2) == "map_passable":
                    lines[i] = "{}placement=map{}\n{}passable=yes\n".format(m.group(1), m.group(3), m.group(1))
                    print('"{}", line {}: placement=map_passable -> placement=map passable=yes'.format(filename,
                                                                                                       i+1))
                elif m.group(2) == "leader_passable":
                    lines[i] = "{}placement=leader{}\n{}passable=yes\n".format(m.group(1), m.group(3), m.group(1))
                    print('"{}", line {}: placement=leader_passable -> placement=leader passable=yes'.format(filename,
                                                                                                             i+1))

    # More syntax transformations would go here.
    return lines

def base_to_base_with_overlay(filename, baseline, inmap, y):
    for i in range(len(inmap[y])):
        for (old, new) in mapchanges:
            inmap[y][i] = inmap[y][i].replace(old, new)
        # this change needs to be handled separately because it's tricky
        # we're converting from a terrain without overlays to a terrain + overlay code
        # since we don't support having two overlays, skip and print a warning
        for (old, new, name) in mapchanges_with_overlay:
            if old + "^" in inmap[y][i]:
                print("{}, line {}: {} ({}) terrain with overlays must be converted manually, cannot add a second overlay".format(filename, baseline + y + 1, old, name))
            elif old in inmap[y][i]:
                inmap[y][i] = inmap[y][i].replace(old, new)

# Generic machinery starts here

tagstack = [] # For tracking tag nesting

def translator(filename, mapxforms, textxform):
    re_map_data = re.compile(r"^\s*map_data\s*=")
    re_mask = re.compile(r"^\s*mask\s*=")
    global tagstack
    map_only = filename.endswith(default_map_extension)

    # Pull file into an array of lines
    try:
        with open(filename, "r", encoding="utf8") as content:
            unmodified = content.read().splitlines()
    except UnicodeDecodeError as e:
        print('wmllint: "{}" is not a valid UTF-8 file'.format(filename), file=sys.stderr)
        return
    
    # array of lines that have potentially been modified
    mfile = []
    # check for UTF8 BOM
    UTF8_BOM = "\ufeff"
    for i, line in enumerate(unmodified):
        # get rid of BOM characters
        if i == 0 and line.startswith(UTF8_BOM):
            line = line[1:]
            print("%s: removed UTF-8 BOM character at the start of the file" % (filename))
        # add all the lines to the new array
        mfile.append(line)

    terminator = "\n"
    lineno = 0
    validate = True
    unbalanced = False
    in_lua_code = False
    in_translatable_attribute = False
    newdata = []

    # Process line-by-line
    while mfile:
        if not map_only:
            line = mfile.pop(0)
            lineno += 1

        # Check for one certain error condition
        if "{" in line and "}" in line:
            refname = line[line.find("{"):line.rfind("}")]
            # Ignore all-caps macro arguments.
            if refname == refname.upper():
                pass
            elif re_mask.search(line) and not (refname.endswith("}") or refname.endswith(default_mask_extension)):
                print('"%s", line %d: mask file without %s extension or not a mask file (%s)' \
                      % (filename, lineno+1, default_mask_extension, refname))

        # Exclude map_data= lines that are just 1 line, entirely empty, referenced via preprocessor inclusion, or part of a [time]
        if map_only or \
            (
                (re_map_data.search(line) or re_mask.search(line))
                and line.count('"') in (1, 2)
                and '""' not in line
                and "{" not in line
                and "}" not in line
                and not within('time')
            ):
            outmap = []
            maptype = None
            baseline = lineno
            in_map_mask_attribute = True

            # determine whether this is map data or mask data
            if map_only:
                if filename.endswith(default_mask_extension):
                    maptype = "mask"
                else:
                    maptype = "map"
            else:
                leadws = leader(line)
                if re_map_data.search(line):
                    maptype = "map"
                elif re_mask.search(line):
                    maptype = "mask"

            # prepare map/mask data in an attribute for parsing
            if not map_only:
                fields = line.split('"')
                if fields[1].strip():
                    mfile.insert(0, fields[1])
                if len(fields) == 3:
                    mfile.insert(1, '"')
            
            # get map/mask data from attribute
            # uses trailing double quote as ending marker
            while in_map_mask_attribute and mfile:
                line = mfile.pop(0)
                lineno += 1
                if '"' in line:
                    in_map_mask_attribute = False
                    line = line.split('"')[0]
                if line:
                    outmap.append(line.split(","))
            
            # re-add the map/mask attribute
            # this causes the map/mask data to always be on the same line as the attribute
            if not map_only:
                if maptype == "map":
                    newdata.append(leadws + "map_data=\"")
                elif maptype == "mask":
                    newdata.append(leadws + "mask=\"")
            
            # execute any map updates, for example base_to_base_with_overlay to convert hexes that went from being a single terrain to a terrain plus an overlay
            for transform in mapxforms:
                for y in range(len(outmap)):
                    transform(filename, baseline, outmap, y)
            
            # concatenate and re-add the map/mask data
            for y in range(len(outmap)):
                newdata.append(",".join(outmap[y]) + terminator)
            
            # add closing quote on separate line
            # this causes the closing quote for map/mask data to always be on its own line
            if not map_only:
                newdata.append("\"" + terminator)
        # if this is map data referenced via preprocessor inclusion
        elif re_map_data.search(line) and ("{" in line or "}" in line):
            newdata.append(line + terminator)
        # if this map data has more than one quote on a single line aka it's an empty map data attribute
        elif re_map_data.search(line) and line.count('"') > 1:
            newdata.append(line + terminator)
        else:
            newline = line

            # translatable attribute start is identified by equals, optional whitespace, underscore, optional whitespace, quote
            if not in_translatable_attribute and re.search(r'=\s*_\s*"', line):
                in_translatable_attribute = True
            
            # code conversions should be skipped in translatable attributes
            if not in_translatable_attribute:
                # Handle text (non-map) lines.  It can use within().
                newline = textxform(filename, lineno, line)
            
            # translatable attribute end is identified by quote, optional whitespace, optional comment, end of line
            if in_translatable_attribute and re.search(r'"\s*(#.*)?$', line):
                in_translatable_attribute = False
            
            newdata.append(newline + terminator)
            fields = newline.split("#")
            trimmed = fields[0]
            destringed = re.sub('"[^"]*"', '', trimmed) # Ignore string literals
            
            # Now do warnings based on the state of the tag stack.
            if not unbalanced and not in_lua_code and re.search(r'^[a-zA-Z0-9_]$', trimmed):
                for instance in re.finditer(r"\[\/?\+?([a-z][a-z_]*[a-z])\]", destringed):
                    tag = instance.group(1)
                    closer = instance.group(0)[1] == '/'

                    if not closer:
                        tagstack.append((tag, {}, []))
                    else:
                        if len(tagstack) == 0:
                            print('"%s", line %d: closer [/%s] with tag stack empty.' % (filename, lineno+1, tag))
                        elif tagstack[-1][0] != tag:
                            print('"%s", line %d: unbalanced [%s] closed with [/%s].' % (filename, lineno+1, tagstack[-1][0], tag))
                        else:
                            if validate:
                                validate_on_pop(tagstack, tag, filename, lineno)
                            tagstack.pop()
                if tagstack:
                    for instance in re.finditer(r'([a-z][a-z_]*[a-z])\s*=(.*)', trimmed):
                        attribute, value = instance.groups()
                        if '#' in value:
                            value = value.split("#")[0]
                        tagstack[-1][1][attribute] = value.strip()

            if "wmllint: validate-on" in newline:
                validate = True
            if "wmllint: validate-off" in newline:
                validate = False
            if "wmllint: unbalanced-on" in newline:
                unbalanced = True
            if "wmllint: unbalanced-off" in newline:
                unbalanced = False

            # single line of Lua code, skip
            if re.match(r"\s*?code\s?=\s?<<.*>>", destringed):
                continue

            # multiline Lua block
            if re.match(r"\s*?code\s?=\s?<<.*", destringed):
                in_lua_code = True
            if in_lua_code and re.match(r".*?>>\s*$", destringed):
                in_lua_code = False

    # It's an error if the tag stack is nonempty at the end of any file:
    if tagstack:
        print('"%s", line %d: tag stack nonempty (%s) at end of file.' % (filename, lineno, tagstack))
    
    tagstack = []
    if iswml(filename):
        # Perform checks that are purely local.  This is an
        # optimization hack to reduce parsing overhead.
        for nav in WmllintIterator(newdata, filename):
            try:
                (key, prefix, value, comment) = parse_attribute(nav.text)
                local_sanity_check(filename, nav, key, prefix, value, comment)
            except TypeError:
                key = prefix = value = comment = None
                local_sanity_check(filename, nav, key, prefix, value, comment)

        # Perform file-global semantic sanity checks
        newdata = global_sanity_check(filename, newdata)

        # OK, now perform WML rewrites
        newdata = hack_syntax(filename, newdata)
        
        # Run everything together
        filetext = "".join(newdata)
        transformed = filetext
    else:
        # Map or mask -- just run everything together
        transformed = "".join(newdata)

    original = terminator.join(unmodified)
    if original[-1] != terminator and transformed[-1] == terminator:
        transformed = transformed[:-1]

    # Return None if the transformation functions made no changes.
    if original != transformed:
        return transformed
    else:
        return None

vctypes = (".svn", ".git", ".hg")

def interesting(path, exclude):
    "Is a file interesting for conversion purposes?"
    if exclude and re.search(exclude, path):
        if verbose:
            print("wmllint: %s excluded" % path, file=sys.stderr)
        return False
    return (path.endswith(default_wml_extension) and not path.endswith("_info.cfg")) or ismap(path) or issave(path)

def allcfgfiles(directory, exclude):
    "Get the names of all interesting files under directory."
    if not os.path.isdir(directory):
        return [os.path.normpath(directory)]

    datafiles = []
    for root, dirs, files in os.walk(directory):
        keep = []
        for dir in dirs:
            path = os.path.join(root, dir)
            if dir in vctypes or (exclude and re.search(exclude, path)):
                if verbose:
                    print("wmllint: %s excluded" % path, file=sys.stderr)
            else:
                keep.append(dir)
        dirs[:] = keep # need to modify dirs in-place so os.walk sees it

        for name in files:
            path = os.path.join(root, name)
            if interesting(path, exclude):
                datafiles.append(path)

    datafiles.sort() # So diffs for same campaigns will cluster in reports
    return map(os.path.normpath, datafiles)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description = '''Convert Battle of Wesnoth WML from older versions to newer ones.
Also validates WML to check for errors.''',
        epilog = '''For more about wmllint, including how to prevent unwanted conversions and false
positive warnings with magic comments, read the introduction in the wmllint
file itself.
See also: https://wiki.wesnoth.org/Maintenance_tools.''',
        formatter_class = argparse.RawTextHelpFormatter,
        prog="wmllint"
    )

    mode = parser.add_mutually_exclusive_group()
    mode.add_argument("-c", "--clean", action="store_true",
                      help="Clean up -bak files.")
    mode.add_argument("-D", "--diffs", action="store_true",
                      help="Display diffs between converted and unconverted files.")
    mode.add_argument("-d", "--dryrun", action="store_true",
                      help="List changes (-v) but don't perform them.")
    mode.add_argument("-r", "--revert", action="store_true",
                      help="Revert the conversion from the -bak files.")
    parser.add_argument("-m", "--missing", action="store_true",
                        help="""Warn about tags without side= keys now applying to all
sides.""")
    parser.add_argument("-s", "--stripcr", action="store_true",
                        help="Convert DOS-style CR/LF to Unix-style LF.")
    parser.add_argument("-v", "--verbose", action="count", default=0,
                        help="""-v        lists changes.
-v -v     names each file before it's processed.
-v -v -v  shows verbose parse details.""")
    parser.add_argument("-K", "--known", action="store_true",
                        help="""Suppress check for unknown unit types, recruits, races,
scenarios, etc.""")
    parser.add_argument("-e", "--exclude", action="append", default=[], help="Takes a regex and excludes files matching it.")
    # -Z --stringfreeze has been removed; it just disabled repairs of double spaces after sentence ends
    # -f --future has been removed; there have been no experimental conversions since 1.4
    # -p --progress has been removed; similar to existing -v -v
    parser.add_argument("directories", action="store", nargs="*",
                        help="Any number of directories. Each directory is converted.")
    parser.add_argument("--version", action="version", version="%(prog)s " + version.as_string)

    namespace = parser.parse_args()
    clean = namespace.clean
    diffs = namespace.diffs
    dryrun = namespace.dryrun
    missingside = namespace.missing
    revert = namespace.revert
    stripcr = namespace.stripcr
    verbose = namespace.verbose
    inconsistency = namespace.known
    exclude = namespace.exclude
    arguments = namespace.directories # a remnant of getopt...

    if not arguments:
        print("No directory specified to run wmllint on")
        exit(1)

    if dryrun:
        verbose = max(1, verbose)

    post15 = False

    def hasdigit(str):
        for c in str:
            if c in "0123456789":
                return True
        return False

    def texttransform(filename, lineno, line):
        "Resource-name transformation on text lines."
        original = line
        # Perform line changes
        if "wmllint: noconvert" not in original:
            for (old, new) in linechanges + mapchanges:
                # code conversions should not affect comments
                if "#" in line:
                    parts = line.split("#", maxsplit=1)
                    parts[0] = parts[0].replace(old, new)
                    line = parts[0] + "#" + parts[1]
                else:
                    line = line.replace(old, new)
            # Perform any base terrain string conversions needed in
            # [terrain_type] aliasof=, mvt_alias=, and def_alias= attributes.
            if under("terrain_type"):
                match = re.search(r"\b(?:aliasof|mvt_alias|def_alias)\s*=(.*)$", line)
                if match:
                    aliases = match.group()
                    for (old, new) in aliaschanges:
                        old = r'\b' + old + r'\b'
                        aliases = re.sub(old, new, aliases)
                    line = line.replace(match.group(), aliases)
        # Perform tag renaming for 1.5.  Note: this has to happen before
        # the sanity check, which assumes [unit] has already been
        # mapped to [unit_type].  Also, beware that this test will fail to
        # convert any unit definitions not in conventionally-named
        # directories -- this is necessary in order to avoid stepping
        # on SingleUnitWML in macro files.  The post15 flag expresses whether
        # we've seen a [unit_type] and can therefore assume the files have
        # undergone 1.4 -> 1.5 conversion.
        global post15
        if "units" in filename and not post15:
            if '[unit_type]' in line:
                post15 = True
            else:
                line = line.replace("[unit]", "[unit_type]")
                line = line.replace("[+unit]", "[+unit_type]")
                line = line.replace("[/unit]", "[/unit_type]")

        # putting the ignore check here to allow setting the post15 flag
        if "wmllint: ignore" in original:
            return original

        # Handle SingleUnitWML or Standard Unit Filter or SideWML
        # Also, when macro calls have description= in them, the arg is
        # a SUF being passed in.
        if tagstack and ((under("unit") and "units" not in filename) or \
               standard_unit_filter() or \
               under("side") or \
               re.search("{[A-Z]+.*description=.*}", line)):
            if "name" not in tagstack[-1][1]:
                line = re.sub(r"user_description\s*=", "name=", line)
            if "generate_name" not in tagstack[-1][1]:
                line = re.sub(r"generate_description\s*=", "generate_name=", line)
        # Now, inside objects...
        if under("object") and "description" not in tagstack[-1][1]:
            line = re.sub(r"user_description\s*=", "description=", line)
        # Alas, WML variable references cannot be converted so
        # automatically.
        if ".description" in line:
            print('"%s", line %d: .description may need hand fixup' % \
                  (filename, lineno))
        if ".user_description" in line:
            print('"%s", line %d: .user_description may need hand fixup' % \
                  (filename, lineno))
        # In unit type definitions
        if under("unit_type") or under("female") or under("unit"):
            line = line.replace("unit_description=", "description=")
            line = line.replace("advanceto=", "advances_to=")
        # Inside themes
        if within("theme"):
            line = line.replace("[unit_description]", "[unit_name]")
        # Report the changes
        if verbose > 0 and line != original:
            msg = "%s, line %d: %s -> %s" % \
                  (filename, lineno, original.strip(), line.strip())
            print(msg)
        return line

    try:
        failed_any_dirs = False

        for directory in arguments:
            if not os.path.exists(directory):
                failed_any_dirs = True
                print("wmllint: skipping non-existent path %s" % directory)
                continue
            for fn in allcfgfiles(directory, "|".join(exclude)):
                if verbose >= 2:
                    print(fn + ":")
                backup = fn + "-bak"
                if clean or revert:
                    # Do housekeeping
                    if os.path.exists(backup):
                        if clean:
                            print("wmllint: removing %s" % backup)
                            if not dryrun:
                                os.remove(backup)
                        elif revert:
                            print("wmllint: reverting %s" % backup)
                            if not dryrun:
                                if sys.platform == 'win32':
                                    os.remove(fn)
                                os.rename(backup, fn)
                elif diffs:
                    # Display diffs
                    if os.path.exists(backup):
                        fromdate = time.ctime(os.stat(backup).st_mtime)
                        todate = time.ctime(os.stat(fn).st_mtime)
                        with open(backup, "r", encoding="utf8") as fromlines, \
                             open(fn, "r", encoding="utf8") as tolines:
                            diff = difflib.unified_diff(fromlines.readlines(),
                                                        tolines.readlines(),
                                                        backup, fn, fromdate, todate, n=3)
                        sys.stdout.writelines(diff)
                else:
                    if "~" in fn:
                        print("wmllint: ignoring %s, the campaign server won't accept it." % fn)
                        continue
                    # Do file conversions
                    try:
                        changed = translator(fn, [base_to_base_with_overlay], texttransform)
                        if changed:
                            print("wmllint: converting", fn)
                            if not dryrun:
                                if sys.platform == 'win32' and os.path.exists(backup):
                                    os.remove(backup)
                                os.rename(fn, backup)
                                with open(fn, "w", encoding="utf8") as ofp:
                                    ofp.write(changed)
                    except:
                        print("wmllint: internal error on %s" % fn, file=sys.stderr)
                        (exc_type, exc_value, exc_traceback) = sys.exc_info()
                        raise exc_type(exc_value).with_traceback(exc_traceback)
        if not clean and not diffs and not revert:
            # Consistency-check everything we got from the file scans
            if not inconsistency:
                consistency_check()
        if failed_any_dirs:
            sys.exit(1)
    except KeyboardInterrupt:
        print("Aborted by pressing ctrl-c")

# wmllint ends here
