Module:Wikitext Parsing

Documentation for this module may be created at Module:Wikitext Parsing/doc

require("strict")

-- We're calling up to thousands of string methods each time we run, so
-- do this very simple micro optimisation to help a tiny bit
local string = string

-- Helper functions for PrepareText --
local function endswith(text, subtext)
	return string.sub(text, -#subtext, -1) == subtext
end
local function allcases(s)
	return string.gsub(s, "%a", function(c)
		return "["..string.upper(c)..string.lower(c).."]"
	end)
end

--[=[ Implementation notes
---- NORMAL HTML TAGS ----
Tags are very strict on how they want to start, but loose on how they end.
The start must strictly follow <[tAgNaMe](%s|>) with no room for whitespace in
the tag's name, but may then flow as they want afterwards, making
<div\nclass\n=\n"\nerror\n"\n> valid. If a tag has no end, it will consume all
text instead of not processing.

There's no sense of escaping < or >
E.g.
 <div class="error\>"> will end at \> despite it being inside a quote
 <div class="<span class="error">error</span>"> will not process the larger div

---- NOPROCESSING TAGS (nowiki, pre, syntaxhighlight, source, etc.) ----
(Note: <source> is the deprecated version of <syntaxhighlight>)

No-processing tags have some differences to the above rules. Specifically, their
syntax is a lot stricter. While an opening tag follows the same set of rules, A
closing tag can't have any sort of extra formatting. </div a/a> is valid,
</nowiki a/a> is not. Only newlines and spaces/tabs are allowed in closing tags.
Note that, even though tags may cause a visual change without an ending tag like
<pre>, one is required for the no-processing effects.

Both the content inside the tag pair and the text in the tags will not be
processed. E.g. <nowiki |}}>|}}</nowiki> would have both of the |}} escaped.

Since we only care about these no-processing tags, we can ignore the idea of an
intercepting tag messing us up, and just go for the first ending we can find. If
there is no ending, the tag will NOT consume the rest of the text. Even if there
is no ending tag, the content inside the opening tag will still be unprocessed,
meaning {{X20|<nowiki }}>}} wouldn't end at the first }} despite there being no
ending tag.

There are some tags, like <math>, which also function like <nowiki> for our
purposes, and are handled here. Some other tags, like <ref>, have more complex
behaviour that can't be reasonably implemented in this, and so are ignored. I
suspect that every tag listed in [[Special:Version]] may behave somewhat like
this, but that's far too many cases worth checking for rarely used tags that may
not even have a good reason to contain {{ or }} anyways, so we leave them alone.

---- INCLUDEONLY ----
While includeonly tags do technically serve the same purpose as a nowiki tag for
what this module does, contextually they don't always make sense to escape, and
as such are ignored by this module entirely.
--]=]
-- This function expects the string to start with the tag
local validtags = {nowiki=1, pre=1, syntaxhighlight=1, source=1, math=1}
local function TestForNowikiTag(text, scanPosition)
	local tagName = string.match(text, "^<([^\n />]+)", scanPosition)
	if not tagName or not validtags[string.lower(tagName)] then
		return nil
	end
	local nextOpener = string.find(text, "<", scanPosition+1, true)
	local nextCloser = string.find(text, ">", scanPosition+1, true)
	if nextCloser and (not nextOpener or nextCloser < nextOpener) then
		local startingTag = string.sub(text, scanPosition, nextCloser)
		-- We have our starting tag (E.g. '<pre style="color:red">')
		-- Now find our ending...
		if endswith(startingTag, "/>") then -- self-closing tag (we are our own ending)
			return {
				Tag = tagName,
				Start = startingTag,
				Content = "", End = "",
				Length = #startingTag
			}

		else
			local endingTagStart, endingTagEnd = string.find(text, "</"..allcases(tagName).."[ \t\n]*>", scanPosition)
			if endingTagStart then --Regular tag formation
				local endingTag = string.sub(text, endingTagStart, endingTagEnd)
				local tagContent = string.sub(text, nextCloser+1, endingTagStart-1)
				return {
					Tag = tagName,
					Start = startingTag,
					Content = tagContent,
					End = endingTag,
					Length = #startingTag + #tagContent + #endingTag
				}

			else -- Content inside still needs escaping (also linter error!)
				return {
					Tag = tagName,
					Start = startingTag,
					Content = "", End = "",
					Length = #startingTag
				}
			end
		end
	end
	return nil
end
--[=[ Implementation Notes
HTML Comments are about as basic as it gets. Start at <!--, end at -->, no extra
conditions. If a comment has no end, it will eat all the text ahead.
--]=]
local function TestForComment(text, scanPosition)
	if string.match(text, "^<!%-%-", scanPosition) then
		local commentEnd = string.find(text, "-->", scanPosition+4, true)
		if commentEnd then
			return {
				Start = "<!--", End = "-->",
				Content = string.sub(text, scanPosition+4, commentEnd-1),
				Length = commentEnd-scanPosition+3
			}
		else -- Consumes all text if not given an ending
			return {
				Start = "<!--", End = "",
				Content = string.sub(text, scanPosition+4),
				Length = #text-scanPosition+1
			}
		end
	end
	return nil
end

--[[ Implementation notes
The goal of this function is to escape all text that wouldn't be parsed if it
was preprocessed (see above implementation notes).

Using keepComments will keep all HTML comments instead of removing them. They
will still be escaped regardless.
--]]
local function PrepareText(text, keepComments)
	local newtext = {}
	local scanPosition = 1
	local textPosition = 1
	while true do
		local nextCheck = string.find(text, "<", scanPosition, true) -- Advance to the next potential tag we care about
		if not nextCheck then -- Done
			newtext[#newtext+1] = string.sub(text, textPosition)
			break
		end
		scanPosition = nextCheck
		local Comment = TestForComment(text, scanPosition)
		if Comment then
			newtext[#newtext+1] = string.sub(text, textPosition, nextCheck-1)
			if keepComments then
				newtext[#newtext+1] = Comment.Start .. mw.text.nowiki(Comment.Content) .. Comment.End
			end
			scanPosition = scanPosition + Comment.Length
			textPosition = scanPosition
		else
			local Tag = TestForNowikiTag(text, scanPosition)
			if Tag then
				newtext[#newtext+1] = string.sub(text, textPosition, nextCheck-1)
				local newTagStart = "<" .. mw.text.nowiki(string.sub(Tag.Start, 2, -2)) .. ">"
				local newTagEnd =
					Tag.End == "" and "" or -- Respect no tag ending
					"</" .. mw.text.nowiki(string.sub(Tag.End, 3, -2)) .. ">"
				local newContent = mw.text.nowiki(Tag.Content)
				newtext[#newtext+1] = newTagStart .. newContent .. newTagEnd
				scanPosition = scanPosition + Tag.Length
				textPosition = scanPosition
			else -- Nothing special, move on...
				scanPosition = scanPosition + 1
			end
		end
	end
	return table.concat(newtext, "")
end

local p = {}
--Main entry points
p.PrepareText = PrepareText
--Extra entry points, not really required
p.TestForNowikiTag = TestForNowikiTag
p.TestForComment = TestForComment

return p

--[==[ console tests

local s = [=[Hey!{{Text|<nowiki | ||>
Hey! }}
A</nowiki>|<!--AAAAA|AAA-->Should see|Shouldn't see}}]=]
local out = p.PrepareText(s)
mw.logObject(out)

local s = [=[B<!--
Hey!
-->A]=]
local out = p.TestForComment(s, 2)
mw.logObject(out); mw.log(string.sub(s, 2, out.Length))

]==]