tszz-client/Lua/Tips/Common/utf8_Tips.lua

---@class utf8
utf8 = {}

--byte index of the next char after the char at byte index i, followed by a valid flag for the char at byte index i.
--nil if not found. invalid characters are iterated as 1-byte chars.
function utf8.next_raw(s, i)
	if not i then
		if #s == 0 then return nil end
		return 1, true --fake flag (doesn't matter since this flag is not to be taken as full validation)
	end
	if i > #s then return end
	local c = s:byte(i)
	if c >= 0x00 and c <= 0x7F then
		i = i + 1
	elseif c >= 0xC2 and c <= 0xDF then
		i = i + 2
	elseif c >= 0xE0 and c <= 0xEF then
		i = i + 3
	elseif c >= 0xF0 and c <= 0xF4 then
		i = i + 4
	else --invalid
		return i + 1, false
	end
	if i > #s then return end
	return i, true
end

--next() is the generic iterator and can be replaced for different semantics. next_raw() must preserve its semantics.
utf8.next = utf8.next_raw

--iterate chars, returning the byte index where each char starts
function utf8.byte_indices(s, previ)
	return utf8.next, s, previ
end

--number of chars in string
function utf8.len(s)
	assert(s, "bad argument #1 to 'len' (string expected, got nil)")
	local len = 0
	for _ in utf8.byte_indices(s) do
		len = len + 1
	end
	return len
end

--byte index given char index. nil if the index is outside the string.
function utf8.byte_index(s, target_ci)
	if target_ci < 1 then return end
	local ci = 0
	for i in utf8.byte_indices(s) do
		ci = ci + 1
		if ci == target_ci then
			return i
		end
	end
	assert(target_ci > ci, "invalid index")
end

--char index given byte index. nil if the index is outside the string.
function utf8.char_index(s, target_i)
	if target_i < 1 or target_i > #s then return end
	local ci = 0
	for i in utf8.byte_indices(s) do
		ci = ci + 1
		if i == target_i then
			return ci
		end
	end
	error("invalid index")
end

--byte index of the prev. char before the char at byte index i, which defaults to #s + 1.
--nil if the index is outside the 2..#s+1 range.
--NOTE: unlike next(), this is a O(N) operation!
function utf8.prev(s, nexti)
	nexti = nexti or #s + 1
	if nexti <= 1 or nexti > #s + 1 then return end
	local lasti, lastvalid = utf8.next(s)
	for i, valid in utf8.byte_indices(s) do
		if i == nexti then
			return lasti, lastvalid
		end
		lasti, lastvalid = i, valid
	end
	if nexti == #s + 1 then
		return lasti, lastvalid
	end
	error("invalid index")
end

--iterate chars in reverse order, returning the byte index where each char starts.
function utf8.byte_indices_reverse(s, nexti)
	if #s < 200 then
		--using prev() is a O(N^2/2) operation, ok for small strings (200 chars need 40,000 iterations)
		return utf8.prev, s, nexti
	else
		--store byte indices in a table and iterate them in reverse.
		--this is 40x slower than byte_indices() but still fast at 2mil chars/second (but eats RAM and makes garbage).
---@class t
---@class t
		local t = {}
		for i in utf8.byte_indices(s) do
			if nexti and i >= nexti then break end
			table.insert(t, i)
		end
		local i = #t + 1
		return function()
			i = i - 1
			return t[i]
		end
	end
end

--sub based on char indices, which, unlike with standard string.sub(), can't be negative.
--start_ci can be 1..inf and end_ci can be 0..inf. end_ci can be nil meaning last char.
--if start_ci is out of range or end_ci < start_ci, the empty string is returned.
--if end_ci is out of range, it is considered to be the last position in the string.
function utf8.sub(s, start_ci, end_ci)
	--assert for positive indices because we might implement negative indices in the future.
	assert(start_ci >= 1)
	assert(not end_ci or end_ci >= 0)
	local ci = 0
	local start_i, end_i
	for i in utf8.byte_indices(s) do
		ci = ci + 1
		if ci == start_ci then
			start_i = i
		end
		if ci == end_ci then
			end_i = i
		end
	end
	if not start_i then
		assert(start_ci > ci, 'invalid index')
		return ''
	end
	if end_ci and not end_i then
		if end_ci < start_ci then
			return ''
		end
		assert(end_ci > ci, 'invalid index')
	end
	return s:sub(start_i, end_i and end_i - 1)
end

--check if a string contains a substring at byte index i without making garbage.
--nil if the index is out of range. true if searching for the empty string.
function utf8.contains(s, i, sub)
	if i < 1 or i > #s then return nil end
	for si = 1, #sub do
		if s:byte(i + si - 1) ~= sub:byte(si) then
			return false
		end
	end
	return true
end

--count the number of occurences of a substring in a string. the substring cannot be the empty string.
function utf8.count(s, sub)
	assert(#sub > 0)
	local count = 0
	local i = 1
	while i do
		if utf8.contains(s, i, sub) then
			count = count + 1
			i = i + #sub
			if i > #s then break end
		else
			i = utf8.next(s, i)
		end
	end
	return count
end

--utf8 validation and sanitization

--check if there's a valid utf8 codepoint at byte index i. valid ranges for each utf8 byte are:
-- byte  1          2           3          4
--------------------------------------------
-- 00 - 7F
-- C2 - DF    80 - BF
-- E0         A0 - BF     80 - BF
-- E1 - EC    80 - BF     80 - BF
-- ED         80 - 9F     80 - BF
-- EE - EF    80 - BF     80 - BF
-- F0         90 - BF     80 - BF    80 - BF
-- F1 - F3    80 - BF     80 - BF    80 - BF
-- F4         80 - 8F     80 - BF    80 - BF
function utf8.isvalid(s, i)
	local c = s:byte(i)
	if not c then
		return false
	elseif c >= 0x00 and c <= 0x7F then
		return true
	elseif c >= 0xC2 and c <= 0xDF then
		local c2 = s:byte(i + 1)
		return c2 and c2 >= 0x80 and c2 <= 0xBF
	elseif c >= 0xE0 and c <= 0xEF then
		local c2 = s:byte(i + 1)
		local c3 = s:byte(i + 2)
		if c == 0xE0 then
			return c2 and c3 and
				c2 >= 0xA0 and c2 <= 0xBF and
				c3 >= 0x80 and c3 <= 0xBF
		elseif c >= 0xE1 and c <= 0xEC then
			return c2 and c3 and
				c2 >= 0x80 and c2 <= 0xBF and
				c3 >= 0x80 and c3 <= 0xBF
		elseif c == 0xED then
			return c2 and c3 and
				c2 >= 0x80 and c2 <= 0x9F and
				c3 >= 0x80 and c3 <= 0xBF
		elseif c >= 0xEE and c <= 0xEF then
			if c == 0xEF and c2 == 0xBF and (c3 == 0xBE or c3 == 0xBF) then
				return false --uFFFE and uFFFF non-characters
			end
			return c2 and c3 and
				c2 >= 0x80 and c2 <= 0xBF and
				c3 >= 0x80 and c3 <= 0xBF
		end
	elseif c >= 0xF0 and c <= 0xF4 then
		local c2 = s:byte(i + 1)
		local c3 = s:byte(i + 2)
		local c4 = s:byte(i + 3)
		if c == 0xF0 then
			return c2 and c3 and c4 and
				c2 >= 0x90 and c2 <= 0xBF and
				c3 >= 0x80 and c3 <= 0xBF and
				c4 >= 0x80 and c4 <= 0xBF
		elseif c >= 0xF1 and c <= 0xF3 then
			return c2 and c3 and c4 and
				c2 >= 0x80 and c2 <= 0xBF and
				c3 >= 0x80 and c3 <= 0xBF and
				c4 >= 0x80 and c4 <= 0xBF
		elseif c == 0xF4 then
			return c2 and c3 and c4 and
				c2 >= 0x80 and c2 <= 0x8F and
				c3 >= 0x80 and c3 <= 0xBF and
				c4 >= 0x80 and c4 <= 0xBF
		end
	end
	return false
end

--byte index of the next valid utf8 char after the char at byte index i.
--nil if indices go out of range. invalid characters are skipped.
function utf8.next_valid(s, i)
	local valid
	i, valid = utf8.next_raw(s, i)
	while i and (not valid or not utf8.isvalid(s, i)) do
		i, valid = utf8.next(s, i)
	end
	return i
end

--iterate valid chars, returning the byte index where each char starts
function utf8.valid_byte_indices(s)
	return utf8.next_valid, s
end

--assert that a string only contains valid utf8 characters
function utf8.validate(s)
	for i, valid in utf8.byte_indices(s) do
		if not valid or not utf8.isvalid(s, i) then
			error(string.format('invalid utf8 char at #%d', i))
		end
	end
end

local function table_lookup(s, i, j, t)
	return t[s:sub(i, j)]
end

--replace characters in string based on a function f(s, i, j, ...) -> replacement_string | nil
function utf8.replace(s, f, ...)
	if type(f) == 'table' then
		return utf8.replace(s, table_lookup, f)
	end
	if s == '' then
		return s
	end
	local t = {}
	local lasti = 1
	for i in utf8.byte_indices(s) do
		local nexti = utf8.next(s, i) or #s + 1
		local repl = f(s, i, nexti - 1, ...)
		if repl then
			table.insert(t, s:sub(lasti, i - 1))
			table.insert(t, repl)
			lasti = nexti
		end
	end
	table.insert(t, s:sub(lasti))
	return table.concat(t)
end

local function replace_invalid(s, i, j, repl_char)
	if not utf8.isvalid(s, i) then
		return repl_char
	end
end

--replace invalid utf8 chars with a replacement char
function utf8.sanitize(s, repl_char)
	repl_char = repl_char or '�' --\uFFFD
	return utf8.replace(s, replace_invalid, repl_char)
end