Handle non-ASCII characters in STEP files

backend/apps/ifc_validation/checks/header_policy/tests/originating_system/pass/x2_escaped_company_name.ifc

@@ -0,0 +1,30 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('ViewDefinition [Alignment-basedView]'),'2;1');
+FILE_NAME('Header example2.ifc', '2022-09-16T10:35:07', ('Evandro Alfieri'), ('buildingSMART Int.'), 'IFC Motor 1.0', 'Buhodra Ingenier\X2\00ED\X0\a S.A. - Istram - 26.06', 'none');
+FILE_SCHEMA(('IFC4X3_ADD2'));
+ENDSEC;
+DATA;
+#1=IFCPERSON($,$,'',$,$,$,$,$);
+#2=IFCORGANIZATION($,'',$,$,$);
+#3=IFCPERSONANDORGANIZATION(#1,#2,$);
+#4=IFCAPPLICATION(#2,'v0.7.0-6c9e130ca','IfcOpenShell-v0.7.0-6c9e130ca','');
+#5=IFCOWNERHISTORY(#3,#4,$,.NOTDEFINED.,$,#3,#4,1700419055);
+#6=IFCDIRECTION((1.,0.,0.));
+#7=IFCDIRECTION((0.,0.,1.));
+#8=IFCCARTESIANPOINT((0.,0.,0.));
+#9=IFCAXIS2PLACEMENT3D(#8,#7,#6);
+#10=IFCDIRECTION((0.,1.));
+#11=IFCGEOMETRICREPRESENTATIONCONTEXT($,'Model',3,1.E-05,#9,#10);
+#12=IFCDIMENSIONALEXPONENTS(0,0,0,0,0,0,0);
+#13=IFCSIUNIT(*,.LENGTHUNIT.,$,.METRE.);
+#14=IFCSIUNIT(*,.AREAUNIT.,$,.SQUARE_METRE.);
+#15=IFCSIUNIT(*,.VOLUMEUNIT.,$,.CUBIC_METRE.);
+#16=IFCSIUNIT(*,.PLANEANGLEUNIT.,$,.RADIAN.);
+#17=IFCMEASUREWITHUNIT(IFCPLANEANGLEMEASURE(0.017453292519943295),#16);
+#18=IFCCONVERSIONBASEDUNIT(#12,.PLANEANGLEUNIT.,'DEGREE',#17);
+#19=IFCUNITASSIGNMENT((#13,#14,#15,#18));
+#20=IFCPROJECT('0iDmeiiLP3AOllitM2Favn',#5,'',$,$,$,$,(#11),#19);
+#21=IFCSITE('3rg2jGkIH10RFhrQsGZKRk',#5,$,$,$,$,$,$,$,$,$,$,$,$);
+ENDSEC;
+END-ISO-10303-21;

backend/apps/ifc_validation/checks/header_policy/validate_header.py

@@ -42,6 +42,51 @@ def ifcopenshell_pre_validation(file):
     return extracted_info
 
 
+# ISO 10303-21 string escapes; \X2\/\X4\ must be tried before the single-byte \X\
+STEP_ESCAPE_PATTERN = re.compile(
+    r"\\X2\\(?P<x2>(?:[0-9A-Fa-f]{4})+)\\X0\\"
+    r"|\\X4\\(?P<x4>(?:[0-9A-Fa-f]{8})+)\\X0\\"
+    r"|\\X\\(?P<x>[0-9A-Fa-f]{2})"
+    r"|\\S\\(?P<s>.)"
+    r"|\\P(?P<p>[A-I])\\"
+    r"|\\(?P<backslash>\\)"
+)
+
+
+def decode_step_string(value):
+    """Decode ISO 10303-21 string escapes (\\X2\\..\\X0\\, \\X\\, \\S\\, \\P?\\, \\\\)
+    so that e.g. 'Ingenier\\X2\\00ED\\X0\\a' is presented as 'Ingeniería'."""
+    if not isinstance(value, str) or "\\" not in value:
+        return value
+
+    parts = []
+    codepage = "iso-8859-1"  # default alphabet; switched by \P?\ directives
+    position = 0
+    for match in STEP_ESCAPE_PATTERN.finditer(value):
+        parts.append(value[position:match.start()])
+        position = match.end()
+        if match.group("x2"):
+            parts.append(bytes.fromhex(match.group("x2")).decode("utf-16-be", "replace"))
+        elif match.group("x4"):
+            parts.append(bytes.fromhex(match.group("x4")).decode("utf-32-be", "replace"))
+        elif match.group("x"):
+            parts.append(bytes([int(match.group("x"), 16)]).decode("iso-8859-1"))
+        elif match.group("s"):
+            parts.append(bytes([(ord(match.group("s")) + 128) & 0xFF]).decode(codepage, "replace"))
+        elif match.group("p"):
+            codepage = f"iso-8859-{ord(match.group('p')) - ord('A') + 1}"
+        else:
+            parts.append("\\")
+    parts.append(value[position:])
+    return "".join(parts)
+
+
+def decode_step_strings(value):
+    if isinstance(value, tuple):
+        return tuple(decode_step_strings(v) for v in value)
+    return decode_step_string(value)
+
+
 def is_valid_iso8601(dt_str: str) -> bool:
     try:
         isoparse(dt_str)
@@ -119,8 +164,8 @@ def populate_header(cls, values):
                 (file_name, 'authorization', 6)
             ]
 
-            attributes = {field: getattr(obj, field) for obj, field, index in fields}
-            
+            attributes = {field: decode_step_strings(getattr(obj, field)) for obj, field, index in fields}
+
             attributes['validation_errors'] = []
             attributes['mvd'] = file.mvd.view_definitions
             attributes['comments'] = file.mvd.comments
@@ -232,11 +277,17 @@ def main():
         print("Usage: python -m validate_header <path_to_ifc_file>")
         sys.exit(1)
 
-    filename = sys.argv[1] 
+    filename = sys.argv[1]
+    # the parse-error class was renamed upstream (SyntaxError -> CollectedValidationErrors);
+    # resolve whichever exists so a parse failure lands in "syntax_error" on either version
+    parse_errors = tuple(
+        exc for name in ("SyntaxError", "CollectedValidationErrors")
+        if (exc := getattr(ifcopenshell.simple_spf, name, None)) is not None
+    )
     try:
         file = ifcopenshell.simple_spf.open(filename, only_header=True)
         header = HeaderStructure(file=file)
-    except ifcopenshell.simple_spf.SyntaxError:
+    except parse_errors:
         header = HeaderStructure(file=None, validation_errors=["syntax_error"])
     except Exception as e:
         print(f"Error opening file '{filename}': {e}")

backend/apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc

@@ -0,0 +1,10 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1');
+FILE_NAME('fail_non_ascii_latin1_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingeniería S.A.'),'ABC rel. 0.1.2','Buhodra Ingeniería S.A. - Istram - 26.06','IFC4 model');
+FILE_SCHEMA(('IFC4'));
+ENDSEC;
+DATA;
+#1=IFCPERSON($,$,'',$,$,$,$,$);
+ENDSEC;
+END-ISO-10303-21;

backend/apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc

@@ -0,0 +1,10 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1');
+FILE_NAME('fail_non_ascii_raw_utf8_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingeniería S.A.'),'ABC rel. 0.1.2','Buhodra Ingeniería S.A. - Istram - 26.06','IFC4 model');
+FILE_SCHEMA(('IFC4'));
+ENDSEC;
+DATA;
+#1=IFCPERSON($,$,'',$,$,$,$,$);
+ENDSEC;
+END-ISO-10303-21;

backend/apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc

@@ -0,0 +1,10 @@
+ISO-10303-21;
+HEADER;
+FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1');
+FILE_NAME('pass_non_ascii_x2_escape_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingenier\X2\00ED\X0\a S.A.'),'ABC rel. 0.1.2','Buhodra Ingenier\X2\00ED\X0\a S.A. - Istram - 26.06','IFC4 model');
+FILE_SCHEMA(('IFC4'));
+ENDSEC;
+DATA;
+#1=IFCPERSON($,$,'',$,$,$,$,$);
+ENDSEC;
+END-ISO-10303-21;

backend/apps/ifc_validation/tasks/processing/syntax.py

@@ -1,9 +1,121 @@
 
 import json
+import re
 
 from apps.ifc_validation_models.models import Model, ValidationOutcome
 from .. import TaskContext, logger, with_model
 
+STEP_ESCAPE_HINT = (
+    "Non-ASCII characters are not allowed in a STEP physical file and must be "
+    "encoded as \\X2\\..\\X0\\ escape sequences (ISO 10303-21)."
+)
+
+MAX_DISPLAY_LINE_LENGTH = 200
+
+UNICODE_DECODE_ERROR_PATTERN = re.compile(
+    r"UnicodeDecodeError: '[^']*' codec can't decode byte (0x[0-9a-fA-F]{2}) in position (\d+)"
+)
+
+# same comment pattern the simple_spf parser blanks out before tokenizing
+COMMENT_PATTERN = re.compile(r"/\*[\s\S]*?\*/")
+
+
+def format_annotated_line(line, column, display_line):
+    if column > MAX_DISPLAY_LINE_LENGTH:
+        # window the display around the column so the caret stays visible
+        start = column - MAX_DISPLAY_LINE_LENGTH // 2
+        display_line = "..." + display_line[start:start + MAX_DISPLAY_LINE_LENGTH]
+        caret_offset = 3 + (column - 1 - start)
+    else:
+        display_line = display_line[:MAX_DISPLAY_LINE_LENGTH]
+        caret_offset = column - 1
+    return f"{line:05d} | {display_line}\n        {' ' * caret_offset}^"
+
+
+def locate_byte_offset(file_path, offset):
+    """Translate a byte offset into (line, column, display_line), reading the file in chunks."""
+    try:
+        line, last_newline_end, bytes_read = 1, 0, 0
+        with open(file_path, "rb") as f:
+            while bytes_read < offset:
+                chunk = f.read(min(1 << 20, offset - bytes_read))
+                if not chunk:
+                    break
+                line += chunk.count(b"\n")
+                newline_at = chunk.rfind(b"\n")
+                if newline_at != -1:
+                    last_newline_end = bytes_read + newline_at + 1
+                bytes_read += len(chunk)
+            column = offset - last_newline_end + 1
+            f.seek(last_newline_end)
+            raw_line = f.read(max(column, MAX_DISPLAY_LINE_LENGTH) + 1).split(b"\n")[0]
+        # latin-1 maps every byte to a character, so the offending line always renders
+        return line, column, raw_line.decode("iso-8859-1")
+    except OSError:
+        return None
+
+
+def locate_first_non_ascii(file_path):
+    """Find the true position of the first non-ASCII character outside comments."""
+    try:
+        with open(file_path, encoding="utf-8", errors="replace") as f:
+            content = f.read()
+    except OSError:
+        return None
+    # blank out comments (preserving newlines) like the parser does: a non-ASCII
+    # character inside a comment never reaches the tokenizer and is not an error
+    blanked = COMMENT_PATTERN.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), content)
+    match = re.search(r"[^\x00-\x7f]", blanked)
+    if not match:
+        return None
+    line = blanked.count("\n", 0, match.start()) + 1
+    line_start = blanked.rfind("\n", 0, match.start()) + 1
+    column = match.start() - line_start + 1
+    # blanking preserves length, so positions in `blanked` map 1:1 onto `content`
+    line_end = content.find("\n", line_start)
+    display_line = content[line_start:line_end if line_end != -1 else None]
+    return line, column, display_line
+
+
+def observed_from_error_output(error_output, file_path):
+    """Build a user-facing message from subprocess stderr, which is never shown raw."""
+    match = UNICODE_DECODE_ERROR_PATTERN.search(error_output)
+    if match:
+        byte, offset = match.group(1), int(match.group(2))
+        located = locate_byte_offset(file_path, offset)
+        if located:
+            line, column, display_line = located
+            return (f"On line {line} column {column}:\n"
+                    f"File contains a non-ASCII byte ('{byte}'). {STEP_ESCAPE_HINT}\n"
+                    f"{format_annotated_line(line, column, display_line)}")
+        return f"File contains a non-ASCII byte ('{byte}') at offset {offset}. {STEP_ESCAPE_HINT}"
+    return "The file could not be parsed as a STEP physical file (ISO 10303-21)."
+
+
+def observed_from_syntax_message(msg, file_path):
+    """Correct the parser's reported position for non-ASCII characters.
+
+    The only_header parser reconstructs the header into a new string before parsing,
+    so its line numbers can point at the wrong line; recompute from the actual file.
+    """
+    message = msg.get("message")
+    if msg.get("type") != "unexpected_character":
+        return message
+    try:
+        found_value = int(msg.get("found_value"), 16)
+    except (TypeError, ValueError):
+        return message
+    if found_value < 0x80:
+        return message
+    located = locate_first_non_ascii(file_path)
+    if not located:
+        return message
+    line, column, display_line = located
+    return (f"On line {line} column {column}:\n"
+            f"Unexpected character ('{msg.get('found_value')}')\n"
+            f"{STEP_ESCAPE_HINT}\n"
+            f"{format_annotated_line(line, column, display_line)}")
+
 
 def process_syntax_outcomes(context:TaskContext):
     #todo - unify output for all task executions
@@ -25,19 +137,19 @@ def process_syntax_outcomes(context:TaskContext):
             task.outcomes.create(
                 severity=ValidationOutcome.OutcomeSeverity.ERROR,
                 outcome_code=ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR,
-                observed=list(filter(None, error_output.split("\n")))[-1]
+                observed=observed_from_error_output(error_output, context.file_path)
             )
         else:
             for msg in json.loads(output):
                 setattr(model, status_field, Model.Status.INVALID)
                 task.outcomes.create(
                     severity=ValidationOutcome.OutcomeSeverity.ERROR,
                     outcome_code=ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR,
-                    observed=msg.get("message")
+                    observed=observed_from_syntax_message(msg, context.file_path)
                 )
 
         model.save(update_fields=[status_field])
-        
+
         # return reason for logging
         return "No IFC syntax error(s)." if success else f"Found IFC syntax errors:\n\nConsole: \n{output}\n\nError: {error_output}"
 
@@ -46,4 +158,4 @@ def process_syntax(context:TaskContext):
     return process_syntax_outcomes(context)
 
 def process_header_syntax(context:TaskContext):
-    return process_syntax_outcomes(context)
+    return process_syntax_outcomes(context)

backend/apps/ifc_validation/tests/tests_header_syntax_validation_task.py

@@ -57,6 +57,80 @@ def test_header_syntax_validation_task_creates_error_validation_outcome(self):
         self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR)
         self.assertTrue('On line 5 column 1' in outcomes.first().observed, outcomes.first().observed)
 
+    def test_header_syntax_validation_task_reports_correct_line_for_raw_utf8(self):
+
+        # the only_header parser reconstructs the header before parsing and used to
+        # report line 3 (FILE_DESCRIPTION) for a non-ASCII character on line 4 (FILE_NAME)
+        SyntaxValidationTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='fail_non_ascii_raw_utf8_header.ifc',
+            file='fail_non_ascii_raw_utf8_header.ifc',
+            size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc'),
+        )
+        request.mark_as_initiated()
+
+        header_syntax_validation_subtask(
+            prev_result={'is_valid': True, 'reason': 'test'},
+            id=request.id,
+            file_name=request.file_name,
+        )
+
+        outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
+        self.assertEqual(len(outcomes), 1)
+        self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.ERROR)
+        self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR)
+        observed = outcomes.first().observed
+        self.assertIn("Unexpected character ('0xed')", observed)
+        self.assertIn('On line 4 column', observed)
+        self.assertNotIn('UnicodeDecodeError', observed)
+
+    def test_header_syntax_validation_task_translates_unicode_decode_error(self):
+
+        SyntaxValidationTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='fail_non_ascii_latin1_header.ifc',
+            file='fail_non_ascii_latin1_header.ifc',
+            size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc'),
+        )
+        request.mark_as_initiated()
+
+        header_syntax_validation_subtask(
+            prev_result={'is_valid': True, 'reason': 'test'},
+            id=request.id,
+            file_name=request.file_name,
+        )
+
+        outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
+        self.assertEqual(len(outcomes), 1)
+        self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.ERROR)
+        observed = outcomes.first().observed
+        self.assertNotIn('UnicodeDecodeError', observed)
+        self.assertNotIn('Traceback', observed)
+        self.assertIn('On line 4 column', observed)
+        self.assertIn('\\X2\\', observed)
+
+    def test_header_syntax_validation_task_passes_x2_escaped_header(self):
+
+        # correctly escaped non-ASCII characters (\X2\00ED\X0\) are valid STEP and must pass
+        SyntaxValidationTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='pass_non_ascii_x2_escape_header.ifc',
+            file='pass_non_ascii_x2_escape_header.ifc',
+            size=os.path.getsize('apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc'),
+        )
+        request.mark_as_initiated()
+
+        header_syntax_validation_subtask(
+            prev_result={'is_valid': True, 'reason': 'test'},
+            id=request.id,
+            file_name=request.file_name,
+        )
+
+        outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id)
+        self.assertEqual(len(outcomes), 1)
+        self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.PASSED)
+        self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.PASSED)
+
     def test_determine_aggregate_status_for_multiple_outcomes(self):
 
         # test cases