From d6c4e9e80ac9ea9b379848a70c7f4c1e7f140066 Mon Sep 17 00:00:00 2001 From: Ian Wrzesinski Date: Sun, 24 May 2026 12:16:00 -0400 Subject: [PATCH] Add and implement the `space-discarding` feature --- Cargo.lock | 205 ++++++++++++++++++ Cargo.toml | 4 +- src/lib.rs | 3 + src/space_discarding.rs | 462 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 673 insertions(+), 1 deletion(-) create mode 100644 src/space_discarding.rs diff --git a/Cargo.lock b/Cargo.lock index e464c497..b41719e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,6 +64,7 @@ name = "codex" version = "0.3.0" dependencies = [ "chinese-number", + "icu_properties", "rustls-webpki", "siphasher", "ureq", @@ -78,6 +79,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -142,6 +154,68 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "itoa" version = "1.0.15" @@ -154,6 +228,12 @@ version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.27" @@ -209,6 +289,15 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -297,6 +386,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "subtle" version = "2.6.1" @@ -314,6 +409,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -362,6 +478,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -459,8 +581,91 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index f35f876a..10dd08ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,12 +12,14 @@ categories = ["encoding", "text-processing"] keywords = ["unicode", "symbols"] [features] -default = ["numeral-systems", "styling"] +default = ["numeral-systems", "styling", "space-discarding"] numeral-systems = ["dep:chinese-number"] styling = [] +space-discarding = ["dep:icu_properties"] _test-unicode-conformance = ["dep:ureq", "dep:rustls-webpki"] [dependencies] +icu_properties = { version = "2.2.0", optional = true } chinese-number = { version = "0.7.7", default-features = false, features = ["number-to-chinese"], optional = true } [dev-dependencies] diff --git a/src/lib.rs b/src/lib.rs index 0ad21d77..eb8e482b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,9 @@ pub mod numeral_systems; #[cfg(feature = "styling")] pub mod styling; +#[cfg(feature = "space-discarding")] +pub mod space_discarding; + /// A module of definitions. #[derive(Debug, Copy, Clone)] pub struct Module(&'static [(&'static str, Binding)]); diff --git a/src/space_discarding.rs b/src/space_discarding.rs new file mode 100644 index 00000000..ccd52fa1 --- /dev/null +++ b/src/space_discarding.rs @@ -0,0 +1,462 @@ +//! Whether to keep or discard spaces that are inferred due to newlines in +//! markup. +//! +//! These definitions allow Chinese and Japanese text to be broken across lines +//! in Typst markup without producing spaces. +//! +//! This is included in Codex for use by other markup systems such as static +//! site generators, Markdown implementations, or HTML renderers. +//! +//! ## Rationale +//! +//! Typst markup should be an ergonomic system to use regardless of which script +//! you are writing in. If there are behaviors Typst applies by default which +//! are obviously wrong in common cases, we endeavor to fix those defaults +//! instead of requiring every user to manually specify an alternative. +//! +//! A common obviously wrong behavior is adding space characters between Chinese +//! and Japanese text split across lines. Chinese and Japanese don't use spaces +//! in writing, but it's still reasonable for authors to format their text files +//! with maximum widths and split text across lines. Applying the rule for Latin +//! text to infer a space from newlines for Chinese and Japanese is incorrect. +//! +//! The straightforward alternative would be to determine whether to keep or +//! discard spaces based solely on a user-provided language tag. But we do not +//! expect authors to tag every individual use of other languages inserted into +//! a main text, which is common in Chinese and Japanese. We may use language +//! tags to improve the current definitions in the future, but we do not +//! consider tags to be a valid replacement for correct default behavior. +//! +//! ## Note on "Writing Systems" +//! +//! Comments in this file use "writing system" as a colloquial term to describe +//! individual scripts or commonly understood collections of scripts. This is +//! because writing out the "Han, Bopomofo, Hiragana, Katakana, and Yi" scripts +//! and distinguishing correctly between Traditional Chinese, Simplified +//! Chinese, and Kanji is a lot of effort. And while this discussion mainly +//! cites Chinese and Japanese for clarity, we endeavor to have ergonomic +//! behavior for all writing systems, scripts, and languages. +//! +//! Much of the research on writing systems used for this feature was based on +//! the wonderfully detailed [Orthography Descriptions][scriptnotes] by Richard +//! Ishida. They proved invaluable here, and are certainly helping his mission +//! of making the World Wide Web worldwide :) +//! +//! [scriptnotes]: https://r12a.github.io/scripts/index.html#scriptnotes +use icu_properties::props::{EastAsianWidth as EAW, Emoji, Script}; +use icu_properties::{CodePointMapDataBorrowed, CodePointSetDataBorrowed}; + +/// Whether to discard an inferred space between two strings. This is `true` +/// when the inner character of _either_ string is unambiguously from a writing +/// system which does not use spaces. Otherwise the inferred space should be +/// kept. +/// +/// Currently this check includes characters which we determine to be from the +/// Chinese, Japanese, or Yi writing systems plus ideographic punctuation. Note +/// that Korean does use spaces between words and predominantly uses Latin +/// punctuation instead of ideographic punctuation. +/// +/// We currently do not discard inferred space characters from writing systems +/// which use spaces only between phrases or sentences. This is because there +/// are not clear distinctions for where phrases or sentences may or may not +/// start/end, so we fall back to keeping all spaces instead of trying to make +/// smart inferences that authors may not expect. +/// +/// We may extend this definition in the future by adding an argument for a +/// language tag to improve the behavior around ambiguous characters. We may +/// also analyze grapheme clusters instead of single characters, but we would +/// only check one grapheme cluster from each string. +#[inline] +pub fn discard_space_between(before: &str, after: &str) -> bool { + // Get the inner characters first since we often need to check both. + let before_c = before.chars().next_back(); + let after_c = after.chars().next(); + // Discard spaces if either character's writing system does not use spaces. + // Keep spaces if both character's writing system uses spaces. + before_c.is_some_and(|c| writing_system_spacing(c) == WritingSystemSpacing::No) + || after_c.is_some_and(|c| writing_system_spacing(c) == WritingSystemSpacing::No) +} + +/// Whether a character is part of a writing system that uses space characters. +/// +/// The only common writing systems which do not use space characters in any +/// respect are Chinese, Japanese, and Yi. +/// +/// These systems do use empty space in writing, but it is empty due to the lack +/// of ink in ideographic characters typeset on a grid (like the ideographic +/// fullstop `。`), not due to explicit space characters in text. +/// +/// Some ideographic punctuation characters primarily used by Chinese, Japanese, +/// and Yi are also used in Korean writing (which uses spaces between words). +/// But these are often used in vertical writing whereas modern Korean +/// predominantly uses Latin punctuation when written horizontally. So this +/// definition considers ideographic punctuation characters as from writing +/// systems which do not use spaces. +/// +/// ## How Writing Systems use Space Characters +/// +/// Beside Chinese, Japanese, and Yi, most writing systems in active use today +/// do use space characters to separate text. Some writing systems use spaces to +/// separate words, some to separate syllables, and some to separate only +/// phrases or sentences. +/// +/// Using spaces between **words** is the most common and includes writing +/// systems such as Latin, Arabic, Cyrillic, Korean, and Hebrew. It also +/// includes many Brahmic (or Indic) scripts such as Devanagari, Bengali, +/// Telugu, Tamil, Gujarati, and Kannada. +/// +/// However, many active writing systems use spaces not between words, but +/// between **phrases and sentences**, sometimes with other punctuation, +/// sometimes without. This includes Thai, Javanese, Burmese, Khmer, Tibetan, +/// and Lao. The Balinese script is likely in this category, but it is hard to +/// find good examples of its use online. +/// +/// Of course, space character frequency for writing systems differs greatly +/// both between systems and within systems when used for different languages. +/// +/// Kinds of space usage do not fall along the common distinction of alphabets, +/// abjads, abugidas, or syllabaries that are normally used to distinguish +/// writing systems. Examples of both word spacing and phrase/sentence spacing +/// are present in both alphabets (Latin/Lao) and abugidas (Devanagari/Thai), +/// although abugidas dominate in phrase/sentence spacing. +/// +/// Finally, one writing system uses spaces uniquely and deserves its own +/// section: +/// +/// ### Ethiopic script (Ge'ez Script) +/// - +/// - +/// +/// The Ethiopic script, or Ge'ez script, is primarily used in the Amharic, +/// Tigrinya, and Tigre languages, and has historically used `፡` U+1361 ETHIOPIC +/// WORDSPACE as a separator between words instead of space characters, but +/// modern text is increasingly using normal spaces instead of the wordspace +/// character. Ethiopic also has several native punctuation characters like `፣` +/// U+1363 ETHIOPIC COMMA and `።` U+1362 ETHIOPIC FULL STOP. A notable +/// complication is that writers very often insert the full stop by writing two +/// wordspace characters instead of using the dedicated codepoint. +/// +/// The wordspace character generally attaches to the end of words when spacing +/// is added (such as for justification), and linebreaks should never separate +/// the wordspace from its preceding character. When using the wordspace +/// character, justified text can insert whitespace either on both sides of the +/// wordspace, centering it between words, or immediately after the wordspace, +/// leaving it connected to the previous word. +/// +/// We should avoid intoducing space characters that the author didn't intend +/// after the wordspace character, but the situation is tricky. A simple rule +/// could be to discard an inferred space which comes after the wordspace +/// character (under the assumption that the wordspace is being used as a +/// space). But this is insufficient as the writer may be using normal spaces +/// instead of wordspaces generally, but still be using pairs of wordspace +/// characters in place of the Ethiopic full stop, after which they would likely +/// expect an inferred space from a newline. +/// +/// Given this complexity, we shouldn't try to automatically discard spaces +/// around Ethiopic script characters, so we currently treat it like a writing +/// system which does use spaces. +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +enum WritingSystemSpacing { + /// Characters whose writing systems do not use space characters to separate + /// text. + /// + /// Only Chinese, Japanese, and Yi do not use spaces. This also includes + /// common ideographic punctuation characters. + /// + /// Amusingly, this also includes U+3000 `　` IDEOGRAPHIC SPACE. + No, + /// Characters whose writing system does use space characters, or characters + /// whose writing system is ambiguous between writing systems which do or + /// don't use spaces (such as the curly quotes U+201C “ and U+201D ” which + /// are used in Latin and in Chinese/Japanese). + /// + /// That includes writing systems which use spaces in any way, such as + /// between syllables, words, phrases, or sentences. + YesOrAmbiguous, +} + +/// Determine the [`WritingSystemSpacing`] for a character. +/// +/// It was very useful to consult the `UnicodeSet` utility when defining this +/// function: . +/// +/// We check the East Asian Width property being `Fullwidth`, `Halfwidth`, or +/// `Wide` to determine common ideographic punctuation characters, unfortunately +/// Unicode reused this property by classifying most emoji characters as `Wide`, +/// so we need to filter those out specifically. More emoji edge cases are +/// discussed in detail in `test_spacing_emoji_presentation` below. +#[inline] +fn writing_system_spacing(c: char) -> WritingSystemSpacing { + const SCRIPT_MAP: CodePointMapDataBorrowed