Module:LocalitySubject

Documentation for this module may be created at Module:LocalitySubject/doc

--[[
Module:LocalitySubject

Parses a locality_subjects= parameter value containing one or more
"Breadcrumb - Subject" lines, copied directly from FamilySearch's Locality
Subject display (hard returns and all), and produces:

  * a deduplicated list of location strings, suitable for Has location
  * a set of category names, one per line, with no cross-product between
    locations and subjects

See Locality_Subject_Module_Spec.md for the full design rationale.

Usage from a template:

  {{#invoke:LocalitySubject|locations|{{{locality_subjects|}}}}}
  {{#invoke:LocalitySubject|categories|{{{locality_subjects|}}}}}

Optional second positional argument on |categories is a suffix appended to
each category (space-joined), used by {{FamilySearch document}} to keep
Source-namespace document categories separate from the Main-namespace
research-source tree (a single volume can spawn many more document pages
than the volume itself, so they need their own branch):

  {{#invoke:LocalitySubject|categories|{{{locality_subjects|}}}|(documents)}}
--]]

local p = {}

-- Countries treated as having at least one confirmed self-contained tier
-- immediately below them (US states, Scottish shires/parishes, Canadian
-- provinces, Australian states, etc.). The country token is dropped from
-- the breadcrumb when present. This is NOT a claim that every place under
-- these countries is self-contained -- exceptions (e.g. England, London)
-- are caught by the override table, which is checked first.
local COUNTRY_DROP_LIST = {
	["United States"] = true,
	["Scotland"] = true,
	["England"] = true,
	["Wales"] = true,
	["Ireland"] = true,
	["Canada"] = true,
	["Australia"] = true,
}

-- US states that don't use "County" for their county-equivalent unit.
-- Applied to the token immediately after the state, since FamilySearch's
-- hierarchy is always Country, State, County-equivalent, [smaller places...]
-- -- that position is reliable regardless of how many more specific tokens
-- follow it.
local US_COUNTY_SUFFIX_BY_STATE = {
	["Louisiana"] = "Parish",
	["Alaska"] = "Borough",
}
local DEFAULT_US_COUNTY_SUFFIX = "County"

local JURISDICTION_WORDS = { "County", "Parish", "Borough", "Census Area", "Municipality" }

-- True if this token needs a jurisdiction word appended -- i.e. it doesn't
-- already carry an explicit qualifier (a parenthetical like
-- "(Independent City)") and doesn't already end in a jurisdiction word.
local function needsJurisdictionSuffix(token)
	if token:find("(", 1, true) then
		return false
	end
	for _, word in ipairs(JURISDICTION_WORDS) do
		if token:sub(-#word) == word then
			return false
		end
	end
	return true
end

-- Scotland: the parish-equivalent token sits right after the shire tier,
-- the same structural position as the US county fix above.
local DEFAULT_SCOTLAND_PARISH_SUFFIX = "Parish"

local function trim(s)
	return mw.text.trim(s or "")
end

-- Split on newlines or semicolons, drop empty lines, trim whitespace.
local function splitInputLines(text)
	text = text or ""
	text = text:gsub("\r\n", "\n"):gsub("\r", "\n")
	-- normalize semicolons to newlines so either delimiter style works
	text = text:gsub(";", "\n")
	local lines = {}
	for line in text:gmatch("[^\n]+") do
		local t = trim(line)
		if t ~= "" then
			table.insert(lines, t)
		end
	end
	return lines
end

local function isBreadcrumb(left)
	if COUNTRY_DROP_LIST[left] then
		return true
	end
	return left:find(",", 1, true) ~= nil
end

-- Convert a raw FamilySearch breadcrumb (e.g. "United States, Virginia,
-- Hampton (Independent City)") into the wiki's location string
-- (e.g. "Hampton (Independent City), Virginia"). Checks
-- {{Locality subject location overrides}} first (a wiki page, kept in the
-- same format/spirit as {{Resource category switch values}} so both are
-- editable without touching Lua); falls back to drop-country-and-reverse
-- when there's no override entry.
--
-- Returns (location, countryRecognized). countryRecognized is false when
-- the leading token wasn't matched by the override table AND wasn't in
-- COUNTRY_DROP_LIST -- e.g. a breadcrumb pasted without its country, which
-- looks structurally identical to a single-tier country (like "Scotland"
-- alone) and would otherwise fail silently rather than triggering the
-- county/parish suffix fix.
local function deriveLocation(frame, left)
	local overridden = frame:expandTemplate{
		title = "Locality subject location overrides",
		args = { left },
	}
	overridden = trim(overridden)
	if overridden ~= "" then
		return overridden, true
	end

	local tokens = {}
	for tok in left:gmatch("[^,]+") do
		table.insert(tokens, trim(tok))
	end

	if #tokens == 0 then
		return nil, false
	end

	local remaining = tokens
	local droppedCountry = nil
	if COUNTRY_DROP_LIST[tokens[1]] and #tokens > 1 then
		droppedCountry = tokens[1]
		remaining = {}
		for i = 2, #tokens do
			table.insert(remaining, tokens[i])
		end
	end

	-- US-specific: fix up the county-equivalent token, which sits right
	-- after the state regardless of how many smaller-place tokens follow it.
	if droppedCountry == "United States" and #remaining >= 2 then
		local state = remaining[1]
		local countyToken = remaining[2]
		if needsJurisdictionSuffix(countyToken) then
			local suffix = US_COUNTY_SUFFIX_BY_STATE[state] or DEFAULT_US_COUNTY_SUFFIX
			remaining[2] = countyToken .. " " .. suffix
		end
	end

	-- Scotland-specific: fix up the parish-equivalent token, which sits
	-- right after the shire tier, same structural position as the US fix.
	if droppedCountry == "Scotland" and #remaining >= 2 then
		local parishToken = remaining[2]
		if needsJurisdictionSuffix(parishToken) then
			remaining[2] = parishToken .. " " .. DEFAULT_SCOTLAND_PARISH_SUFFIX
		end
	end

	-- reverse
	local reversed = {}
	for i = #remaining, 1, -1 do
		table.insert(reversed, remaining[i])
	end

	-- countryRecognized covers two legitimate cases: a country was actually
	-- dropped, OR the breadcrumb is genuinely a single bare country/state
	-- name with nothing more specific (#tokens == 1) -- there's nothing
	-- wrong with those, just nothing to flag. What's NOT legitimate is a
	-- multi-token breadcrumb whose first token isn't a recognized country
	-- at all -- that's the missing-"United States"-style mistake. (A
	-- single unrecognized token never reaches here, since isBreadcrumb()
	-- only accepts it if it's a known country or contains a comma.)
	local countryRecognized = (droppedCountry ~= nil) or (#tokens == 1)

	return table.concat(reversed, ", "), countryRecognized
end

-- Split a single line into (location-or-nil, subject-string, countryRecognized).
-- location is nil when the line has no recognizable place breadcrumb
-- (a bare subject, as used on non-cataloged {{Book source}} entries).
-- countryRecognized is always true for bare-subject lines (nothing to flag).
local function parseLine(frame, line)
	local dashPos = line:find(" %- ")
	if not dashPos then
		return nil, line, true
	end

	local left = trim(line:sub(1, dashPos - 1))
	local right = trim(line:sub(dashPos + 3))

	if isBreadcrumb(left) then
		local location, countryRecognized = deriveLocation(frame, left)
		return location, right, countryRecognized
	else
		return nil, line, true
	end
end

-- Look up the subject in {{Resource category switch values}} via the
-- existing template, so that table stays the single source of truth for
-- subject-name normalization (including the reordering already applied to
-- entries like census years and military/war subjects).
local function normalizeSubject(frame, subject)
	local result = frame:expandTemplate{
		title = "Resource category switch values",
		args = { subject },
	}
	result = trim(result)
	if result == "" then
		-- Not found in the switch table -- surface this clearly rather than
		-- silently emitting a blank/missing category.
		return nil
	end
	return result
end

local function joinCategory(location, subjectNormalized, suffix)
	local base
	if location == nil then
		base = subjectNormalized
	elseif location:find(",", 1, true) then
		base = location .. ", " .. subjectNormalized
	else
		base = location .. " " .. subjectNormalized
	end
	if suffix and suffix ~= "" then
		base = base .. " " .. suffix
	end
	return base
end

-- Shared parse across both entry points, so |locations and |categories stay
-- in sync from a single pass over the input.
local function parseAll(frame, text)
	local lines = splitInputLines(text)
	local locations = {}
	local seenLocations = {}
	local categories = {}
	local missingSubjects = {}
	local unrecognizedCountryLines = {}

	for _, line in ipairs(lines) do
		local location, subject, countryRecognized = parseLine(frame, line)

		if location and not seenLocations[location] then
			seenLocations[location] = true
			table.insert(locations, location)
		end

		if not countryRecognized then
			table.insert(unrecognizedCountryLines, line)
		end

		local normalized = normalizeSubject(frame, subject)
		if normalized then
			table.insert(categories, { location = location, subject = normalized, rawSubject = subject })
		else
			table.insert(missingSubjects, subject)
		end
	end

	return locations, categories, missingSubjects, unrecognizedCountryLines
end

-- {{#invoke:LocalitySubject|locations|{{{locality_subjects|}}}}}
-- Returns a semicolon-separated list of location strings, for feeding into
-- ArrayMapPropertyBulletedListLink / Has location, exactly as a hand-typed
-- location= parameter would have.
function p.locations(frame)
	local text = frame.args[1] or ""
	local locations = parseAll(frame, text)
	return table.concat(locations, ";")
end

-- {{#invoke:LocalitySubject|subjects|{{{locality_subjects|}}}}}
-- Returns a semicolon-separated list of RAW subject terms (FamilySearch's
-- own text, e.g. "Land and property"), NOT run through the switch table.
-- This preserves the previous Has document type semantics, which took the
-- raw locality_subjects= value directly -- other queries on the wiki that
-- filter on Has document type expect FamilySearch's own vocabulary, not the
-- wiki's lowercased category-name text.
function p.subjects(frame)
	local text = frame.args[1] or ""
	local _, categories = parseAll(frame, text)
	local subjects = {}
	local seen = {}
	for _, entry in ipairs(categories) do
		if not seen[entry.rawSubject] then
			seen[entry.rawSubject] = true
			table.insert(subjects, entry.rawSubject)
		end
	end
	return table.concat(subjects, ";")
end

-- {{#invoke:LocalitySubject|categories|{{{locality_subjects|}}}|optionalSuffix}}
-- Returns wikitext category tags, one per parsed line. If a subject isn't
-- found in the switch table, that line is skipped (no blank/broken
-- category) and a tracking category is added so the gap is visible rather
-- than silent. Likewise, if a line's leading breadcrumb token isn't a
-- recognized country (e.g. "United States" was left off by accident), a
-- separate tracking category is added -- the location/category still get
-- generated with the best guess available, but flagged for review since
-- the county/parish suffix fix never had a chance to run.
function p.categories(frame)
	local text = frame.args[1] or ""
	local suffix = frame.args[2]
	local _, categories, missingSubjects, unrecognizedCountryLines = parseAll(frame, text)

	local out = {}
	for _, entry in ipairs(categories) do
		local catName = joinCategory(entry.location, entry.subject, suffix)
		table.insert(out, "[[Category:" .. catName .. "]]")
	end

	if #missingSubjects > 0 then
		table.insert(out, "[[Category:Locality subjects needing switch table entries]]")
	end

	if #unrecognizedCountryLines > 0 then
		table.insert(out, "[[Category:Locality subjects missing recognized country]]")
	end

	return table.concat(out, "")
end

return p