From e1881af6cc360bad03b38bb2955ce375a5d36c1d Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 20 Mar 2026 18:37:59 -0400 Subject: [PATCH 1/5] Pass **kwargs for encoding Also, add CharacterEncoding="ASCII" in test helper --- mathics/format/box/makeboxes.py | 13 ++++++++----- test/builtin/assignments/test_assignment.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mathics/format/box/makeboxes.py b/mathics/format/box/makeboxes.py index 4df7cbf1e..9e8775d45 100644 --- a/mathics/format/box/makeboxes.py +++ b/mathics/format/box/makeboxes.py @@ -76,7 +76,10 @@ def eval_makeboxes_traditional_form(expr, evaluation): def apply_makeboxes_rules( - expr: BaseElement, evaluation: Evaluation, form: Symbol = SymbolStandardForm + expr: BaseElement, + evaluation: Evaluation, + form: Symbol = SymbolStandardForm, + **kwargs, ) -> BoxElementMixin: """ This function takes the definitions provided by the evaluation @@ -124,7 +127,7 @@ def yield_rules(): if parent_form is not form: return apply_makeboxes_rules(expr, evaluation, parent_form) - return eval_generic_makeboxes(expr, form, evaluation) + return eval_generic_makeboxes(expr, form, evaluation, **kwargs) # TODO: evaluation is needed because `atom_to_boxes` uses it. Can we remove this @@ -205,7 +208,7 @@ def eval_makeboxes_fullform_recursive( return RowBox(*result_elements) -def eval_generic_makeboxes(expr, f, evaluation): +def eval_generic_makeboxes(expr, f, evaluation, **kwargs): """MakeBoxes[expr_, f:TraditionalForm|StandardForm]""" from mathics.builtin.box.layout import RowBox @@ -224,7 +227,7 @@ def eval_generic_makeboxes(expr, f, evaluation): printform_callback = PRINT_FORMS_CALLBACK.get(head.get_name(), None) if printform_callback is not None: - return printform_callback(elements[0], evaluation) + return printform_callback(elements[0], evaluation, **kwargs) f_name = f.get_name() if f_name == "System`TraditionalForm": @@ -296,7 +299,7 @@ def format_element( if form not in evaluation.definitions.boxforms: formatted_expr = Expression(form, formatted_expr) form = SymbolStandardForm - result_box = apply_makeboxes_rules(formatted_expr, evaluation, form) + result_box = apply_makeboxes_rules(formatted_expr, evaluation, form, **kwargs) if isinstance(result_box, BoxElementMixin): return result_box return eval_makeboxes_fullform_recursive(element, evaluation) diff --git a/test/builtin/assignments/test_assignment.py b/test/builtin/assignments/test_assignment.py index 877a300f9..def799c3c 100644 --- a/test/builtin/assignments/test_assignment.py +++ b/test/builtin/assignments/test_assignment.py @@ -8,7 +8,7 @@ """ # TODO: consider to split this module in sub-modules. -from test.helper import check_evaluation, session +from test.helper import check_evaluation import pytest from mathics_scanner.errors import IncompleteSyntaxError From 1c4c369e2b8a208f27cb1b923e40f51bd156d49d Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 23 Mar 2026 12:47:52 -0400 Subject: [PATCH 2/5] Finish propagating CharacterEncoding into Forms --- mathics/format/box/formatvalues.py | 56 +++++++++++---------- mathics/format/box/makeboxes.py | 2 +- mathics/format/box/outputforms.py | 16 ++++-- test/builtin/assignments/test_assignment.py | 2 +- test/builtin/test_comparison.py | 2 +- test/helper.py | 4 +- 6 files changed, 47 insertions(+), 35 deletions(-) diff --git a/mathics/format/box/formatvalues.py b/mathics/format/box/formatvalues.py index 90210bcfd..d4d9cb666 100644 --- a/mathics/format/box/formatvalues.py +++ b/mathics/format/box/formatvalues.py @@ -5,7 +5,6 @@ formatting rules. """ - from typing import Any, Callable, Dict, List, Optional, Type from mathics.core.atoms import Complex, Integer, Rational, String, SymbolI @@ -48,17 +47,23 @@ def do_format( - element: BaseElement, evaluation: Evaluation, form: Symbol + element: BaseElement, + evaluation: Evaluation, + form: Symbol, + encoding: Optional[str] = None, ) -> BaseElement: do_format_method = _element_formatters.get(type(element), do_format_element) - result = do_format_method(element, evaluation, form) + result = do_format_method(element, evaluation, form, encoding) if result is None: return element return result def do_format_element( - element: BaseElement, evaluation: Evaluation, form: Symbol + element: BaseElement, + evaluation: Evaluation, + form: Symbol, + encoding: Optional[str] = None, ) -> Optional[BaseElement]: """ Applies formats associated to the expression and removes @@ -135,7 +140,7 @@ def format_expr(expr): formatted = format_expr(expr) if isinstance(expr, EvalMixin) else None if formatted is not None: do_format_fn = _element_formatters.get(type(formatted), do_format_element) - result = do_format_fn(formatted, evaluation, form) + result = do_format_fn(formatted, evaluation, form, encoding) if include_form and result is not None: result = Expression(form, result) return result @@ -153,7 +158,7 @@ def format_expr(expr): if len(expr.get_elements()) != 1: return expr do_format_fn = _element_formatters.get(type(element), do_format_element) - result = do_format_fn(expr, evaluation, form) + result = do_format_fn(expr, evaluation, form, encoding) if isinstance(result, Expression): expr = result @@ -172,7 +177,7 @@ def format_expr(expr): ) expr_head = expr.head do_format = _element_formatters.get(type(expr_head), do_format_element) - head = do_format(expr_head, evaluation, form) or expr_head + head = do_format(expr_head, evaluation, form, encoding) or expr_head expr = to_expression_with_specialization(head, *new_elements) if include_form: @@ -183,7 +188,10 @@ def format_expr(expr): def do_format_rational( - element: BaseElement, evaluation: Evaluation, form: Symbol + element: BaseElement, + evaluation: Evaluation, + form: Symbol, + encoding: Optional[str] = None, ) -> Optional[BaseElement]: if not isinstance(element, Rational): return None @@ -197,12 +205,15 @@ def do_format_rational( if minus: result = Expression(SymbolMinus, result) result = Expression(SymbolHoldForm, result) - result = do_format_expression(result, evaluation, form) or result + result = do_format_expression(result, evaluation, form, encoding) or result return result def do_format_complex( - element: BaseElement, evaluation: Evaluation, form: Symbol + element: BaseElement, + evaluation: Evaluation, + form: Symbol, + encoding: Optional[str] = None, ) -> Optional[BaseElement]: if not isinstance(element, Complex): return None @@ -220,27 +231,18 @@ def do_format_complex( else: result = Expression(SymbolPlus, *parts) - return do_format_expression(Expression(SymbolHoldForm, result), evaluation, form) + return do_format_expression( + Expression(SymbolHoldForm, result), evaluation, form, encoding + ) def do_format_expression( - element: BaseElement, evaluation: Evaluation, form: Symbol + element: BaseElement, + evaluation: Evaluation, + form: Symbol, + encoding: Optional[str] = None, ) -> BaseElement: - # # not sure how much useful is this format_cache - # if element._format_cache is None: - # element._format_cache = {} - - # last_evaluated_time, expr = element._format_cache.get(form, (None, None)) - # if last_evaluated_time is not None and expr is not None: - # if True - # symbolname = expr.get_name() - # if symbolname != "": - # if not evaluation.definitions.is_uncertain_final_value( - # last_evaluated_time, set((symbolname,)) - # ): - # return expr - expr = do_format_element(element, evaluation, form) or element - # element._format_cache[form] = (evaluation.definitions.now, expr) + expr = do_format_element(element, evaluation, form, encoding) or element return expr diff --git a/mathics/format/box/makeboxes.py b/mathics/format/box/makeboxes.py index 9e8775d45..755ba2126 100644 --- a/mathics/format/box/makeboxes.py +++ b/mathics/format/box/makeboxes.py @@ -295,7 +295,7 @@ def format_element( Applies formats associated to the expression, and then calls Makeboxes """ evaluation.is_boxing = True - formatted_expr = do_format(element, evaluation, form) + formatted_expr = do_format(element, evaluation, form, **kwargs) if form not in evaluation.definitions.boxforms: formatted_expr = Expression(form, formatted_expr) form = SymbolStandardForm diff --git a/mathics/format/box/outputforms.py b/mathics/format/box/outputforms.py index a139828d8..ebf67d629 100644 --- a/mathics/format/box/outputforms.py +++ b/mathics/format/box/outputforms.py @@ -1,4 +1,5 @@ import re +from typing import Optional from mathics.core.atoms import Integer, String from mathics.core.element import BaseElement, BoxElementMixin @@ -24,7 +25,9 @@ @is_print_form_callback("System`MathMLForm") -def eval_mathmlform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixin: +def eval_mathmlform( + expr: BaseElement, evaluation: Evaluation, encoding: Optional[str] = None +) -> BoxElementMixin: "MakeBoxes[MathMLForm[expr_], form_]" from mathics.builtin.box.layout import InterpretationBox @@ -63,7 +66,12 @@ def eval_mathmlform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixi def eval_tableform( - self, table: BaseElement, f: Symbol, evaluation: Evaluation, options + self, + table: BaseElement, + f: Symbol, + evaluation: Evaluation, + options, + encoding: Optional[str] = None, ): """MakeBoxes[TableForm[table_], f_]""" from mathics.builtin.box.layout import GridBox @@ -124,7 +132,9 @@ def transform_item(item): @is_print_form_callback("System`TeXForm") -def eval_texform(expr: BaseElement, evaluation: Evaluation) -> BoxElementMixin: +def eval_texform( + expr: BaseElement, evaluation: Evaluation, encoding: Optional[str] = None +) -> BoxElementMixin: from mathics.builtin.box.layout import InterpretationBox boxes = format_element(expr, evaluation, SymbolTraditionalForm) diff --git a/test/builtin/assignments/test_assignment.py b/test/builtin/assignments/test_assignment.py index def799c3c..877a300f9 100644 --- a/test/builtin/assignments/test_assignment.py +++ b/test/builtin/assignments/test_assignment.py @@ -8,7 +8,7 @@ """ # TODO: consider to split this module in sub-modules. -from test.helper import check_evaluation +from test.helper import check_evaluation, session import pytest from mathics_scanner.errors import IncompleteSyntaxError diff --git a/test/builtin/test_comparison.py b/test/builtin/test_comparison.py index dc40168e6..2a95e8091 100644 --- a/test/builtin/test_comparison.py +++ b/test/builtin/test_comparison.py @@ -113,7 +113,7 @@ def test_compare_many_members( ): # if str_expr is None: # reset_session() - result = session.evaluate(f"ToString[{str_expr}]").value + result = session.evaluate(f'ToString[{str_expr}, CharacterEncoding->"ASCII"]').value print("result:", result) if assert_fail_message: assert result == str_expected, assert_fail_message diff --git a/test/helper.py b/test/helper.py index dc0900eb4..9da776d36 100644 --- a/test/helper.py +++ b/test/helper.py @@ -110,7 +110,7 @@ def check_evaluation( str_expected = "Null" if to_string_expr: - str_expr = f"ToString[{str_expr}]" + str_expr = f'ToString[{str_expr}, CharacterEncoding->"ASCII"]' result = evaluate_value(str_expr) elif to_string_expr is None: result = str_expr @@ -123,7 +123,7 @@ def check_evaluation( if hold_expected: expected = str_expected else: - str_expected = f"ToString[{str_expected}]" + str_expected = f'ToString[{str_expected}, CharacterEncoding->"ASCII"]' expected = evaluate_value(str_expected) elif to_string_expected is None: expected = str_expected From b1aa8b1430b5857579fe6f93ef2084aa571b4c45 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 23 Mar 2026 12:53:06 -0400 Subject: [PATCH 3/5] Update type annotaion signature --- mathics/format/box/formatvalues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mathics/format/box/formatvalues.py b/mathics/format/box/formatvalues.py index d4d9cb666..2ec2fe757 100644 --- a/mathics/format/box/formatvalues.py +++ b/mathics/format/box/formatvalues.py @@ -42,7 +42,7 @@ _element_formatters: Dict[ Type[BaseElement], - Callable[[BaseElement, Evaluation, Symbol], Optional[BaseElement]], + Callable[[BaseElement, Evaluation, Symbol, Optional[str]], Optional[BaseElement]], ] = {} @@ -170,7 +170,7 @@ def format_expr(expr): new_elements = tuple( ( _element_formatters.get(type(element), do_format_element)( - element, evaluation, form + element, evaluation, form, encoding ) for element in expr.elements ) From 4786a329b90355044fd7e45be533e5a175e03968 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 24 Mar 2026 12:53:24 -0400 Subject: [PATCH 4/5] Go over ToString doc examples... In particular, add examples with the CharacterEncoding option. --- mathics/builtin/atomic/strings.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mathics/builtin/atomic/strings.py b/mathics/builtin/atomic/strings.py index 81bd10052..aaa46320b 100644 --- a/mathics/builtin/atomic/strings.py +++ b/mathics/builtin/atomic/strings.py @@ -866,8 +866,24 @@ class ToString(Builtin): >> ToString[2] = 2 + + Notice how the output changes when we switch formatting from 'StandardForm', the default, to \ + 'InputForm': + >> ToString[2] // InputForm = "2" + + 'ToString' can act a translator of expressions to one expression format to another: + >> ToString[Integrate[f[x],x], TeXForm] + = \\int f(x) \\, dx + + 'ToString' also handles character encoding changes, when passed the 'CharacterEncoding' option: + >> ToString[a >= b, CharacterEncoding-> "UTF-8"] + = a ≥ b + + >> ToString[a ≥ b, CharacterEncoding-> "ASCII"] + = a >= b + >> ToString[a+b] = a + b >> "U" <> 2 @@ -875,8 +891,6 @@ class ToString(Builtin): = U <> 2 >> "U" <> ToString[2] = U2 - >> ToString[Integrate[f[x],x], TeXForm] - = \\int f(x) \\, dx """ From 8ccf13bf24062f601826227ef11fd74f34c4de47 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 24 Mar 2026 18:20:40 -0400 Subject: [PATCH 5/5] Comment out a docstest that fails on MS Windows... due to a character code page problem --- mathics/builtin/atomic/strings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mathics/builtin/atomic/strings.py b/mathics/builtin/atomic/strings.py index aaa46320b..6f53deeaf 100644 --- a/mathics/builtin/atomic/strings.py +++ b/mathics/builtin/atomic/strings.py @@ -881,8 +881,9 @@ class ToString(Builtin): >> ToString[a >= b, CharacterEncoding-> "UTF-8"] = a ≥ b - >> ToString[a ≥ b, CharacterEncoding-> "ASCII"] - = a >= b + ## Reinstate when we've worked out how to handle MS-Windows + ## >> ToString[a ≥ b, CharacterEncoding-> "ASCII"] + ## = a >= b >> ToString[a+b] = a + b