Add support for ambiguous-width characters

Add a new ambiguous_width parameter to the wcwidth() function, which
allows indicating which width should be reported for such characters.
By default it will return 1, but can be changed to 2 for those cases
where needed.

While at it, move table generation to a new generate-tables Lua script,
which better handles input and character attributes.

Author: aperezdc

README.md

@@ -136,6 +136,7 @@ With them, it generates the following files:
 
 * [wcwidth/widetab.lua](./wcwidth/widetab.lua)
 * [wcwidth/zerotab.lua](./wcwidth/zerotab.lua)
+* [wcwidth/ambitab.lua](./wcwidth/ambitab.lua)
 
 The most current version of `wcwidth` uses the following versions of the above
 Unicode Standard release files:

README.md.in

@@ -136,6 +136,7 @@ With them, it generates the following files:
 
 * [wcwidth/widetab.lua](./wcwidth/widetab.lua)
 * [wcwidth/zerotab.lua](./wcwidth/zerotab.lua)
+* [wcwidth/ambitab.lua](./wcwidth/ambitab.lua)
 
 The most current version of `wcwidth` uses the following versions of the above
 Unicode Standard release files:

generate-tables

@@ -0,0 +1,114 @@
+#! /usr/bin/env lua
+-- vim:set ft=lua:sw=3:ts=3:
+
+local help_text = [[Usage: %s [-w<path>] [-z<path>] [-a<path>] [-p<path>] [--help | -h]
+
+  -w<path>    Path to wide characters table (default: widetab.lua)
+  -z<path>    Path to zero-width characters table (default: zerotab.lua)
+  -a<path>    Path to ambiguous width characters table (default: ambitab.lua)
+  -p<path>    Path prefix to prepend to all above paths (default: .)
+  -h, --help  Show this help text.
+
+]]
+
+local path_prefix = "."
+local wide_tab_path = "widetab.lua"
+local zero_tab_path = "zerotab.lua"
+local ambi_tab_path = "ambitab.lua"
+
+for _, item in ipairs(arg) do
+	if item == "-h" or item == "--help" then
+		io.stdout:write(help_text:format(arg[0]))
+		os.exit(0)
+	end
+	local prefix = item:sub(1, 2)
+	local suffix = item:sub(3)
+	if prefix == "-w" then
+		wide_tab_path = suffix
+	elseif prefix == "-z" then
+		zero_tab_path = suffix
+	elseif prefix == "-a" then
+		ambi_tab_path = suffix
+	elseif prefix == "-p" then
+		path_prefix = suffix
+	else
+		io.stderr:write(("Unrecognized command line option: %s\n\n"):format(item))
+		io.stderr:write(help_text:format(arg[0]))
+		os.exit(1)
+	end
+end
+
+local pat_range = "^(%x+)%.%.(%x+)%s*;%s*(%w+)"
+local pat_rune  = "^(%x+)%s*;%s*(%w+)"
+
+local wide_tab = {}
+local zero_tab = {}
+local ambi_tab = {}
+
+local wide_attrs = {
+	F = true,
+	W = true,
+}
+local zero_attrs = {
+	Cf = true,
+	Mc = true,
+	Me = true,
+	Mn = true,
+	Zl = true,
+	Zp = true,
+}
+local ambi_attrs = {
+	A = true,
+}
+
+for line in io.lines() do
+	local range_start, range_end, attribute, range_start_s, range_end_s
+
+	range_start_s, attribute = line:match(pat_rune)
+	if range_start_s then
+		range_start = tonumber(range_start_s, 16)
+		range_end = range_start
+	else
+		range_start_s, range_end_s, attribute = line:match(pat_range)
+		if range_start_s then
+			range_start = tonumber(range_start_s, 16)
+			range_end = tonumber(range_end_s, 16)
+		end
+	end
+
+	if range_start then
+		local tab
+		if wide_attrs[attribute] then
+			tab = wide_tab
+		elseif zero_attrs[attribute] then
+			tab = zero_tab
+		elseif ambi_attrs[attribute] then
+			tab = ambi_tab
+		end
+		if tab then
+			tab[#tab + 1] = { range_start, range_end }
+		end
+	end
+end
+
+local function tab_sort_compare(a, b)
+	return a[1] < b[1]
+end
+
+table.sort(wide_tab, tab_sort_compare)
+table.sort(zero_tab, tab_sort_compare)
+table.sort(ambi_tab, tab_sort_compare)
+
+local tab_dump_line_format = "\t0x%X, 0x%X,\n"
+local function tab_dump(tab, out)
+	out:write("-- Automatically generated, do not edit\n")
+	out:write("return {\n")
+	for _, item in ipairs(tab) do
+		out:write(tab_dump_line_format:format(item[1], item[2]))
+	end
+	out:write("}\n")
+end
+
+tab_dump(wide_tab, io.open(path_prefix .. "/" .. wide_tab_path, "w"))
+tab_dump(zero_tab, io.open(path_prefix .. "/" .. zero_tab_path, "w"))
+tab_dump(ambi_tab, io.open(path_prefix .. "/" .. ambi_tab_path, "w"))

spec/wcwidth_spec.lua

@@ -25,10 +25,15 @@ local function test_phrase(input, expected_lengths, expected_total_length)
 end
 
 --
--- Many test cases are from:
--- https://github.com/jquast/wcwidth/blob/master/wcwidth/tests/test_core.py
+-- A number of test cases are from:
+-- https://github.com/jquast/wcwidth/blob/master/tests/
 --
 describe("wcwidth()", function ()
+   it("reports double-width for Katakana Ko", function ()
+      local input = "コ"
+      local rune = utf8.codepoint(input)
+      assert.equal(2, wcwidth(rune))
+   end)
    it("handles a mix of Japanese and ASCII", function ()
       -- Given a phrase of 5 and 3 Katakana ideographs, joined with 3 English
       -- ASCII punctuation characters, totaling 11, this phrase consumes 19
@@ -54,16 +59,21 @@ describe("wcwidth()", function ()
       -- Phrase cafe + COMBINING ACUTE ACCENT is café of length 4.
       test_phrase("cafe" .. utf8.char(0x0301), { 1, 1, 1, 1, 0 }, 4)
    end)
-   it("handles a combining enclosing", function ()
-      -- CYRILLIC CAPITAL LETTER A + COMBINING CYRILLIC HUNDRED THOUSANDS SIGN is А҈ of length 1.
-      test_phrase(utf8.char(0x0410, 0x0488), { 1, 0 }, 1)
+   it("handles a combining enclosing character", function ()
+      -- CAPITAL LETTER A + COMBINING ENCLOSING CIRCLE has length 1.
+      test_phrase(utf8.char(0x41, 0x20DD), { 1, 0 }, 1)
    end)
-   it("handles combining spaces", function ()
-      -- Balinese kapal (ship) is ᬓᬨᬮ᭄ of length 4.
-      test_phrase(utf8.char(0x1B13, 0x1B28, 0x1B2E, 0x1B44), { 1, 1, 1, 1 }, 4)
+   it("handles multiple combining characters", function ()
+      -- A + acute + grave
+      test_phrase(utf8.char(0x41, 0x0301, 0x0300), { 1, 0, 0 }, 1)
    end)
    it("handles a 👍 emoji", function ()
       test_phrase("two 👍", { 1, 1, 1, 1, 2 }, 6)
    end)
+   it("can report ambiguous-width char as either 1 or 2", function ()
+      assert.equal(1, wcwidth(0x451))
+      assert.equal(1, wcwidth(0x451, 1))
+      assert.equal(2, wcwidth(0x451, 2))
+   end)
 end)
 

update-tables

@@ -22,78 +22,20 @@ ZERO_FILE="${srcdir}/${ZERO_URL##*/}"
 [[ -r ${WIDE_FILE} ]] || wget -c -O "${WIDE_FILE}" "${WIDE_URL}"
 [[ -r ${ZERO_FILE} ]] || wget -c -O "${ZERO_FILE}" "${ZERO_URL}"
 
-# Pad hex values to be 8 characters wide, and prepend "0x" to them.
-u32hex () {
-	local -i len=${#1}
-	local -i nzeros=$(( 8 - len ))
-	echo -n '0x'
-	for (( ; nzeros > 0 ; nzeros-- )) ; do
-		echo -n '0'
-	done
-	echo -n "$1"
-}
-
-format_range () {
-	if [[ $1 = *..* ]] ; then
-		u32hex "${1%..*}"
-		echo -n ','
-		u32hex "${1#*..}"
-		echo ','
-	else
-		u32hex "${1}"
-		echo -n ','
-		u32hex "${1}"
-		echo ','
-	fi
-}
-
-zero_table () {
-	local tmpfile="${TMPDIR:-/tmp}/update-tables-$$-${RANDOM}"
-	echo "-- Autogenerated from ${ZERO_FILE##*/}"
-	echo 'return {'
-	while read -r line ; do
-		if [[ -z ${line} || ${line} = \#* ]] ; then
-			read line rest <<< "${line}"
-			if [[ ${rest} = *.txt ]] ; then
-				ZERO_VER=${rest}
-			elif [[ ${rest} = Date:\ 20* || ${rest} = ©\ 20* ]] ; then
-				ZERO_VER="${ZERO_VER}, ${rest}"
-			fi
-			continue
-		fi
-		read -a items <<< "${line}"
-		if [[ ${items[2]} == Mn || ${items[2]} == Me ]] ; then
-			format_range "${items[0]}"
-		fi
-	done > "${tmpfile}"
-	sort "${tmpfile}"
-	rm "${tmpfile}"
-	echo '}'
-}
-
-wide_table () {
-	local tmpfile="${TMPDIR:-/tmp}/update-tables-$$-${RANDOM}"
-	echo "-- Autogenerated from ${WIDE_FILE##*/}"
-	echo 'return {'
+parse_file_version () {
+	local version rest line
 	while read -r line ; do
 		if [[ -z ${line} || ${line} = \#* ]] ; then
 			read -r line rest <<< "${line}"
 			if [[ ${rest} = *.txt ]] ; then
-				WIDE_VER=${rest}
+				version=${rest}
 			elif [[ ${rest} = Date:\ 20* || ${rest} = ©\ 20* ]] ; then
-				WIDE_VER="${WIDE_VER}, ${rest}"
+				version="${version}, ${rest}"
 			fi
 			continue
 		fi
-		read -r -a items <<< "${line}"
-		case ${items[2]} in
-			F | W | A )
-				format_range "${items[0]}" ;;
-		esac
-	done > "${tmpfile}"
-	sort "${tmpfile}"
-	rm "${tmpfile}"
-	echo '}'
+	done
+	echo "${version}"
 }
 
 make_readme () {
@@ -109,6 +51,8 @@ make_readme () {
 		-e "s+@@LUAROCKS_VER@@+${V}+g"
 }
 
-wide_table  < "${WIDE_FILE}" > "${srcdir}/wcwidth/widetab.lua"
-zero_table  < "${ZERO_FILE}" > "${srcdir}/wcwidth/zerotab.lua"
+WIDE_VER=$(parse_file_version < "${WIDE_FILE}")
+ZERO_VER=$(parse_file_version < "${ZERO_FILE}")
 make_readme < "${srcdir}/README.md.in" > "${srcdir}/README.md"
+
+cat "${WIDE_FILE}" "${ZERO_FILE}" | "${srcdir}/generate-tables" -p"${srcdir}/wcwidth"