Merge remote-tracking branch 'Bjoern-Ge/feature/unicode-superscripts' into iss1676

Author: sangwinc

stack/cas/parsingrules/180_char_based_superscripts.filter.php

@@ -37,10 +37,18 @@ class stack_ast_filter_180_char_based_superscripts implements stack_cas_astfilte
     // phpcs:ignore moodle.Commenting.MissingDocblock.Function
     public function filter(MP_Node $ast, array &$errors, array &$answernotes, stack_cas_security $identifierrules): MP_Node {
         if (self::$ssmap === null) {
-            self::$ssmap = json_decode(file_get_contents(__DIR__ . '/../../maximaparser/unicode/superscript-stack.json'), true);
+            // Option C: Allow only the most common superscripts: ², ³, ¹
+            // These are 2-byte UTF-8 characters and work without UTF-8mb4 requirement.
+            self::$ssmap = ['²' => '2', '³' => '3', '¹' => '1'];
         }
 
-        $process = function ($node) use (&$errors, &$answernotes) {
+        // Loop until no more changes are made.
+        // This ensures all superscripts are converted, even in expressions like x²+x².
+        $changed = true;
+        while ($changed) {
+            $changed = false;
+
+            $process = function($node) use (&$errors, &$answernotes, &$changed) {
             if ($node instanceof MP_Identifier && !(isset($node->position['invalid']) && $node->position['invalid'])) {
                 // Iterate over the name to detect when we move from normal to superscript.
                 $norm = true;
@@ -108,6 +116,7 @@ public function filter(MP_Node $ast, array &$errors, array &$answernotes, stack_
 
                     if (count($parts) === 1) {
                         $node->parentnode->replace($node, $parts[0]);
+                        $changed = true;
                     } else {
                         if (array_search('missing_stars', $answernotes) === false) {
                             $answernotes[] = 'missing_stars';
@@ -119,6 +128,7 @@ public function filter(MP_Node $ast, array &$errors, array &$answernotes, stack_
                             $a->position['insertstars'] = true;
                         }
                         $node->parentnode->replace($node, $a);
+                        $changed = true;
                     }
                     return false;
                 }
@@ -127,6 +137,7 @@ public function filter(MP_Node $ast, array &$errors, array &$answernotes, stack_
         };
 
         $ast->callbackRecurse($process);
+        }
         return $ast;
     }
 }

stack/input/inputbase.class.php

@@ -1043,6 +1043,10 @@ protected function validate_contents_filters($basesecurity) {
             $filterstoapply[] = '115_lexer_post_process_stackbasen';
         }
 
+        // Filter 180 MUST run BEFORE Filter 150!
+        // Filter 180: Intelligently converts Unicode superscripts to exponents (x² → x^2)
+        // Filter 150: Simple replacement of remaining Unicode letters (α → alpha, × → *)
+        $filterstoapply[] = '180_char_based_superscripts';
         $filterstoapply[] = '150_replace_unicode_letters';
 
         if (get_class($this) === 'stack_units_input' || get_class($this) === 'stack_numerical_input') {

stack/maximaparser/corrective_parser.php

@@ -118,7 +118,12 @@ public static function parse(string $string, array &$errors, array &$answernote,
 
         // Check for invalid chars at this point as they may prove to be difficult to
         // handle latter, also strings are safe already.
+        // Option C: Allow only the most common superscripts: ², ³, ¹
+        // These are 2-byte UTF-8 characters and work without UTF-8mb4 requirement.
+        $superscript = ['²' => '2', '³' => '3', '¹' => '1'];
+
         $allowedcharsregex = '~[^' . preg_quote(
+            implode('', array_keys($superscript)) .
             // @codingStandardsIgnoreStart
             // We do really want a backtick here.
             '0123456789,./\%#&{}[]()$@!"\'?`^~*_+qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM:;=><|: -', '~'

tests/fixtures/inputfixtures.class.php

@@ -520,8 +520,8 @@ function at a point \(f(x)\).  Maybe a 'gocha' for the question author....",
         ['(x/y)/z', 'php_true', '(x/y)/z', 'cas_true', '\frac{\frac{x}{y}}{z}', '', ""],
         ['x/(y/z)', 'php_true', 'x/(y/z)', 'cas_true', '\frac{x}{\frac{y}{z}}', '', ""],
         ['x^y', 'php_true', 'x^y', 'cas_true', 'x^{y}', '', "Operations and functions with special TeX"],
-        ["x\u{00b2}", 'php_false', '', 'cas_false', '', 'forbiddenChar', ""],
-        ["x\u{00b2}*x\u{00b2}", 'php_false', '', 'cas_false', '', 'forbiddenChar', ""],
+        ["x\u{00b2}", 'php_true', 'x^2', 'cas_true', 'x^{2}', 'superscriptchars', ""],
+        ["x\u{00b2}*x\u{00b2}", 'php_true', 'x^2*x^2', 'cas_true', 'x^{2}\cdot x^{2}', 'superscriptchars', ""],
         ['x^(y+z)', 'php_true', 'x^(y+z)', 'cas_true', 'x^{y+z}', '', ""],
         ['x^(y/z)', 'php_true', 'x^(y/z)', 'cas_true', 'x^{\frac{y}{z}}', '', ""],
         ['x^f(x)', 'php_true', 'x^f(x)', 'cas_true', 'x^{f\left(x\right)}', '', ""],
@@ -625,7 +625,7 @@ function at a point \(f(x)\).  Maybe a 'gocha' for the question author....",
         ['arsinh(x)', 'php_true', 'asinh(x)', 'cas_true', '{\rm sinh}^{-1}\left( x \right)', 'triginv', ""],
         ['sin^-1(x)', 'php_false', 'sin^-1(x)', 'cas_false', '', 'missing_stars | trigexp', ""],
         ['cos^2(x)', 'php_false', 'cos^2(x)', 'cas_false', '', 'missing_stars | trigexp', ""],
-        ["sin\u{00b2}(x)", 'php_false', 'sin^2(x)', 'cas_false', '', 'forbiddenChar', ""],
+        ["sin\u{00b2}(x)", 'php_false', 'sin^2(x)', 'cas_false', '', 'trigexp | superscriptchars | forbiddenVariable', ""],
         ['sin*2*x', 'php_false', 'sin*2*x', 'cas_false', '', 'forbiddenVariable', ""],
         ['sin[2*x]', 'php_false', 'sin[2*x]', 'cas_false', '', 'trigparens', ""],
         ['cosh(x)', 'php_true', 'cosh(x)', 'cas_true', '\cosh \left( x \right)', '', ""],

tests/fixtures/test_strings.json

@@ -1,4 +1,13 @@
 [
+    "x²",
+    "x³",
+    "x¹",
+    "x²²",
+    "x²³",
+    "x²+x²",
+    "x²+x²+x²",
+    "x²y³",
+    "(x²)+(y²)",
     "\"+\"(a,b)",
     "\"1+1\"",
     "\"Hello world\"",

tests/input_algebraic_test.php

@@ -3197,20 +3197,16 @@ public function test_validate_student_response_single_var_chars_unicode_superscr
         $el->set_parameter('insertStars', 2);
         $state = $el->validate_student_response(
             ['sans1' => 'x²'],
-            $options,
-            'x^2',
+            $options, 'x^2',
             new stack_cas_security()
         );
-        $this->assertEquals(stack_input::INVALID, $state->status);
-        // The rest needs to be updated once we know what the expected result is.
-        $this->assertEquals('forbiddenChar', $state->note);
-        $this->assertEquals(
-            'CAS commands may not contain the following characters: ².',
-            $state->errors
-        );
-        $this->assertEquals('', $state->contentsmodified);
-        $this->assertEquals('<span class="stacksyntaxexample">x&sup2;</span>', $state->contentsdisplayed);
-        $this->assertEquals('', $state->lvars);
+        // Unicode superscripts are now converted to exponents by Filter 180.
+        $this->assertEquals(stack_input::VALID, $state->status);
+        $this->assertEquals('superscriptchars', $state->note);
+        $this->assertEquals('', $state->errors);
+        $this->assertEquals('x^2', $state->contentsmodified);
+        $this->assertEquals('\[ x^{2} \]', $state->contentsdisplayed);
+        $this->assertEquals('x', $state->lvars);
     }
 
     public function test_validate_student_response_km(): void {