diff --git a/backend/apps/ifc_validation/checks/header_policy/tests/originating_system/pass/x2_escaped_company_name.ifc b/backend/apps/ifc_validation/checks/header_policy/tests/originating_system/pass/x2_escaped_company_name.ifc new file mode 100644 index 00000000..a4ec8eab --- /dev/null +++ b/backend/apps/ifc_validation/checks/header_policy/tests/originating_system/pass/x2_escaped_company_name.ifc @@ -0,0 +1,30 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('ViewDefinition [Alignment-basedView]'),'2;1'); +FILE_NAME('Header example2.ifc', '2022-09-16T10:35:07', ('Evandro Alfieri'), ('buildingSMART Int.'), 'IFC Motor 1.0', 'Buhodra Ingenier\X2\00ED\X0\a S.A. - Istram - 26.06', 'none'); +FILE_SCHEMA(('IFC4X3_ADD2')); +ENDSEC; +DATA; +#1=IFCPERSON($,$,'',$,$,$,$,$); +#2=IFCORGANIZATION($,'',$,$,$); +#3=IFCPERSONANDORGANIZATION(#1,#2,$); +#4=IFCAPPLICATION(#2,'v0.7.0-6c9e130ca','IfcOpenShell-v0.7.0-6c9e130ca',''); +#5=IFCOWNERHISTORY(#3,#4,$,.NOTDEFINED.,$,#3,#4,1700419055); +#6=IFCDIRECTION((1.,0.,0.)); +#7=IFCDIRECTION((0.,0.,1.)); +#8=IFCCARTESIANPOINT((0.,0.,0.)); +#9=IFCAXIS2PLACEMENT3D(#8,#7,#6); +#10=IFCDIRECTION((0.,1.)); +#11=IFCGEOMETRICREPRESENTATIONCONTEXT($,'Model',3,1.E-05,#9,#10); +#12=IFCDIMENSIONALEXPONENTS(0,0,0,0,0,0,0); +#13=IFCSIUNIT(*,.LENGTHUNIT.,$,.METRE.); +#14=IFCSIUNIT(*,.AREAUNIT.,$,.SQUARE_METRE.); +#15=IFCSIUNIT(*,.VOLUMEUNIT.,$,.CUBIC_METRE.); +#16=IFCSIUNIT(*,.PLANEANGLEUNIT.,$,.RADIAN.); +#17=IFCMEASUREWITHUNIT(IFCPLANEANGLEMEASURE(0.017453292519943295),#16); +#18=IFCCONVERSIONBASEDUNIT(#12,.PLANEANGLEUNIT.,'DEGREE',#17); +#19=IFCUNITASSIGNMENT((#13,#14,#15,#18)); +#20=IFCPROJECT('0iDmeiiLP3AOllitM2Favn',#5,'',$,$,$,$,(#11),#19); +#21=IFCSITE('3rg2jGkIH10RFhrQsGZKRk',#5,$,$,$,$,$,$,$,$,$,$,$,$); +ENDSEC; +END-ISO-10303-21; diff --git a/backend/apps/ifc_validation/checks/header_policy/validate_header.py b/backend/apps/ifc_validation/checks/header_policy/validate_header.py index b0aa74fb..7ae2dcb4 100644 --- a/backend/apps/ifc_validation/checks/header_policy/validate_header.py +++ b/backend/apps/ifc_validation/checks/header_policy/validate_header.py @@ -42,6 +42,51 @@ def ifcopenshell_pre_validation(file): return extracted_info +# ISO 10303-21 string escapes; \X2\/\X4\ must be tried before the single-byte \X\ +STEP_ESCAPE_PATTERN = re.compile( + r"\\X2\\(?P(?:[0-9A-Fa-f]{4})+)\\X0\\" + r"|\\X4\\(?P(?:[0-9A-Fa-f]{8})+)\\X0\\" + r"|\\X\\(?P[0-9A-Fa-f]{2})" + r"|\\S\\(?P.)" + r"|\\P(?P

[A-I])\\" + r"|\\(?P\\)" +) + + +def decode_step_string(value): + """Decode ISO 10303-21 string escapes (\\X2\\..\\X0\\, \\X\\, \\S\\, \\P?\\, \\\\) + so that e.g. 'Ingenier\\X2\\00ED\\X0\\a' is presented as 'Ingeniería'.""" + if not isinstance(value, str) or "\\" not in value: + return value + + parts = [] + codepage = "iso-8859-1" # default alphabet; switched by \P?\ directives + position = 0 + for match in STEP_ESCAPE_PATTERN.finditer(value): + parts.append(value[position:match.start()]) + position = match.end() + if match.group("x2"): + parts.append(bytes.fromhex(match.group("x2")).decode("utf-16-be", "replace")) + elif match.group("x4"): + parts.append(bytes.fromhex(match.group("x4")).decode("utf-32-be", "replace")) + elif match.group("x"): + parts.append(bytes([int(match.group("x"), 16)]).decode("iso-8859-1")) + elif match.group("s"): + parts.append(bytes([(ord(match.group("s")) + 128) & 0xFF]).decode(codepage, "replace")) + elif match.group("p"): + codepage = f"iso-8859-{ord(match.group('p')) - ord('A') + 1}" + else: + parts.append("\\") + parts.append(value[position:]) + return "".join(parts) + + +def decode_step_strings(value): + if isinstance(value, tuple): + return tuple(decode_step_strings(v) for v in value) + return decode_step_string(value) + + def is_valid_iso8601(dt_str: str) -> bool: try: isoparse(dt_str) @@ -119,8 +164,8 @@ def populate_header(cls, values): (file_name, 'authorization', 6) ] - attributes = {field: getattr(obj, field) for obj, field, index in fields} - + attributes = {field: decode_step_strings(getattr(obj, field)) for obj, field, index in fields} + attributes['validation_errors'] = [] attributes['mvd'] = file.mvd.view_definitions attributes['comments'] = file.mvd.comments @@ -232,11 +277,17 @@ def main(): print("Usage: python -m validate_header ") sys.exit(1) - filename = sys.argv[1] + filename = sys.argv[1] + # the parse-error class was renamed upstream (SyntaxError -> CollectedValidationErrors); + # resolve whichever exists so a parse failure lands in "syntax_error" on either version + parse_errors = tuple( + exc for name in ("SyntaxError", "CollectedValidationErrors") + if (exc := getattr(ifcopenshell.simple_spf, name, None)) is not None + ) try: file = ifcopenshell.simple_spf.open(filename, only_header=True) header = HeaderStructure(file=file) - except ifcopenshell.simple_spf.SyntaxError: + except parse_errors: header = HeaderStructure(file=None, validation_errors=["syntax_error"]) except Exception as e: print(f"Error opening file '{filename}': {e}") diff --git a/backend/apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc b/backend/apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc new file mode 100644 index 00000000..b06cb01b --- /dev/null +++ b/backend/apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc @@ -0,0 +1,10 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1'); +FILE_NAME('fail_non_ascii_latin1_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingeniería S.A.'),'ABC rel. 0.1.2','Buhodra Ingeniería S.A. - Istram - 26.06','IFC4 model'); +FILE_SCHEMA(('IFC4')); +ENDSEC; +DATA; +#1=IFCPERSON($,$,'',$,$,$,$,$); +ENDSEC; +END-ISO-10303-21; diff --git a/backend/apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc b/backend/apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc new file mode 100644 index 00000000..1a68d8c0 --- /dev/null +++ b/backend/apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc @@ -0,0 +1,10 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1'); +FILE_NAME('fail_non_ascii_raw_utf8_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingeniería S.A.'),'ABC rel. 0.1.2','Buhodra Ingeniería S.A. - Istram - 26.06','IFC4 model'); +FILE_SCHEMA(('IFC4')); +ENDSEC; +DATA; +#1=IFCPERSON($,$,'',$,$,$,$,$); +ENDSEC; +END-ISO-10303-21; diff --git a/backend/apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc b/backend/apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc new file mode 100644 index 00000000..0974128f --- /dev/null +++ b/backend/apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc @@ -0,0 +1,10 @@ +ISO-10303-21; +HEADER; +FILE_DESCRIPTION(('ViewDefinition [ReferenceView_V1.2]', 'ExchangeRequirement [Any]'),'2;1'); +FILE_NAME('pass_non_ascii_x2_escape_header.ifc','2025-02-13T15:58:45',('jdoe'),('Buhodra Ingenier\X2\00ED\X0\a S.A.'),'ABC rel. 0.1.2','Buhodra Ingenier\X2\00ED\X0\a S.A. - Istram - 26.06','IFC4 model'); +FILE_SCHEMA(('IFC4')); +ENDSEC; +DATA; +#1=IFCPERSON($,$,'',$,$,$,$,$); +ENDSEC; +END-ISO-10303-21; diff --git a/backend/apps/ifc_validation/tasks/processing/syntax.py b/backend/apps/ifc_validation/tasks/processing/syntax.py index 112cbab9..460bd9cb 100644 --- a/backend/apps/ifc_validation/tasks/processing/syntax.py +++ b/backend/apps/ifc_validation/tasks/processing/syntax.py @@ -1,9 +1,121 @@ import json +import re from apps.ifc_validation_models.models import Model, ValidationOutcome from .. import TaskContext, logger, with_model +STEP_ESCAPE_HINT = ( + "Non-ASCII characters are not allowed in a STEP physical file and must be " + "encoded as \\X2\\..\\X0\\ escape sequences (ISO 10303-21)." +) + +MAX_DISPLAY_LINE_LENGTH = 200 + +UNICODE_DECODE_ERROR_PATTERN = re.compile( + r"UnicodeDecodeError: '[^']*' codec can't decode byte (0x[0-9a-fA-F]{2}) in position (\d+)" +) + +# same comment pattern the simple_spf parser blanks out before tokenizing +COMMENT_PATTERN = re.compile(r"/\*[\s\S]*?\*/") + + +def format_annotated_line(line, column, display_line): + if column > MAX_DISPLAY_LINE_LENGTH: + # window the display around the column so the caret stays visible + start = column - MAX_DISPLAY_LINE_LENGTH // 2 + display_line = "..." + display_line[start:start + MAX_DISPLAY_LINE_LENGTH] + caret_offset = 3 + (column - 1 - start) + else: + display_line = display_line[:MAX_DISPLAY_LINE_LENGTH] + caret_offset = column - 1 + return f"{line:05d} | {display_line}\n {' ' * caret_offset}^" + + +def locate_byte_offset(file_path, offset): + """Translate a byte offset into (line, column, display_line), reading the file in chunks.""" + try: + line, last_newline_end, bytes_read = 1, 0, 0 + with open(file_path, "rb") as f: + while bytes_read < offset: + chunk = f.read(min(1 << 20, offset - bytes_read)) + if not chunk: + break + line += chunk.count(b"\n") + newline_at = chunk.rfind(b"\n") + if newline_at != -1: + last_newline_end = bytes_read + newline_at + 1 + bytes_read += len(chunk) + column = offset - last_newline_end + 1 + f.seek(last_newline_end) + raw_line = f.read(max(column, MAX_DISPLAY_LINE_LENGTH) + 1).split(b"\n")[0] + # latin-1 maps every byte to a character, so the offending line always renders + return line, column, raw_line.decode("iso-8859-1") + except OSError: + return None + + +def locate_first_non_ascii(file_path): + """Find the true position of the first non-ASCII character outside comments.""" + try: + with open(file_path, encoding="utf-8", errors="replace") as f: + content = f.read() + except OSError: + return None + # blank out comments (preserving newlines) like the parser does: a non-ASCII + # character inside a comment never reaches the tokenizer and is not an error + blanked = COMMENT_PATTERN.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), content) + match = re.search(r"[^\x00-\x7f]", blanked) + if not match: + return None + line = blanked.count("\n", 0, match.start()) + 1 + line_start = blanked.rfind("\n", 0, match.start()) + 1 + column = match.start() - line_start + 1 + # blanking preserves length, so positions in `blanked` map 1:1 onto `content` + line_end = content.find("\n", line_start) + display_line = content[line_start:line_end if line_end != -1 else None] + return line, column, display_line + + +def observed_from_error_output(error_output, file_path): + """Build a user-facing message from subprocess stderr, which is never shown raw.""" + match = UNICODE_DECODE_ERROR_PATTERN.search(error_output) + if match: + byte, offset = match.group(1), int(match.group(2)) + located = locate_byte_offset(file_path, offset) + if located: + line, column, display_line = located + return (f"On line {line} column {column}:\n" + f"File contains a non-ASCII byte ('{byte}'). {STEP_ESCAPE_HINT}\n" + f"{format_annotated_line(line, column, display_line)}") + return f"File contains a non-ASCII byte ('{byte}') at offset {offset}. {STEP_ESCAPE_HINT}" + return "The file could not be parsed as a STEP physical file (ISO 10303-21)." + + +def observed_from_syntax_message(msg, file_path): + """Correct the parser's reported position for non-ASCII characters. + + The only_header parser reconstructs the header into a new string before parsing, + so its line numbers can point at the wrong line; recompute from the actual file. + """ + message = msg.get("message") + if msg.get("type") != "unexpected_character": + return message + try: + found_value = int(msg.get("found_value"), 16) + except (TypeError, ValueError): + return message + if found_value < 0x80: + return message + located = locate_first_non_ascii(file_path) + if not located: + return message + line, column, display_line = located + return (f"On line {line} column {column}:\n" + f"Unexpected character ('{msg.get('found_value')}')\n" + f"{STEP_ESCAPE_HINT}\n" + f"{format_annotated_line(line, column, display_line)}") + def process_syntax_outcomes(context:TaskContext): #todo - unify output for all task executions @@ -25,7 +137,7 @@ def process_syntax_outcomes(context:TaskContext): task.outcomes.create( severity=ValidationOutcome.OutcomeSeverity.ERROR, outcome_code=ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR, - observed=list(filter(None, error_output.split("\n")))[-1] + observed=observed_from_error_output(error_output, context.file_path) ) else: for msg in json.loads(output): @@ -33,11 +145,11 @@ def process_syntax_outcomes(context:TaskContext): task.outcomes.create( severity=ValidationOutcome.OutcomeSeverity.ERROR, outcome_code=ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR, - observed=msg.get("message") + observed=observed_from_syntax_message(msg, context.file_path) ) model.save(update_fields=[status_field]) - + # return reason for logging return "No IFC syntax error(s)." if success else f"Found IFC syntax errors:\n\nConsole: \n{output}\n\nError: {error_output}" @@ -46,4 +158,4 @@ def process_syntax(context:TaskContext): return process_syntax_outcomes(context) def process_header_syntax(context:TaskContext): - return process_syntax_outcomes(context) \ No newline at end of file + return process_syntax_outcomes(context) diff --git a/backend/apps/ifc_validation/tests/tests_header_syntax_validation_task.py b/backend/apps/ifc_validation/tests/tests_header_syntax_validation_task.py index b637ff2c..010852ef 100644 --- a/backend/apps/ifc_validation/tests/tests_header_syntax_validation_task.py +++ b/backend/apps/ifc_validation/tests/tests_header_syntax_validation_task.py @@ -57,6 +57,80 @@ def test_header_syntax_validation_task_creates_error_validation_outcome(self): self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR) self.assertTrue('On line 5 column 1' in outcomes.first().observed, outcomes.first().observed) + def test_header_syntax_validation_task_reports_correct_line_for_raw_utf8(self): + + # the only_header parser reconstructs the header before parsing and used to + # report line 3 (FILE_DESCRIPTION) for a non-ASCII character on line 4 (FILE_NAME) + SyntaxValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='fail_non_ascii_raw_utf8_header.ifc', + file='fail_non_ascii_raw_utf8_header.ifc', + size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc'), + ) + request.mark_as_initiated() + + header_syntax_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name, + ) + + outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id) + self.assertEqual(len(outcomes), 1) + self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.ERROR) + self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR) + observed = outcomes.first().observed + self.assertIn("Unexpected character ('0xed')", observed) + self.assertIn('On line 4 column', observed) + self.assertNotIn('UnicodeDecodeError', observed) + + def test_header_syntax_validation_task_translates_unicode_decode_error(self): + + SyntaxValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='fail_non_ascii_latin1_header.ifc', + file='fail_non_ascii_latin1_header.ifc', + size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc'), + ) + request.mark_as_initiated() + + header_syntax_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name, + ) + + outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id) + self.assertEqual(len(outcomes), 1) + self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.ERROR) + observed = outcomes.first().observed + self.assertNotIn('UnicodeDecodeError', observed) + self.assertNotIn('Traceback', observed) + self.assertIn('On line 4 column', observed) + self.assertIn('\\X2\\', observed) + + def test_header_syntax_validation_task_passes_x2_escaped_header(self): + + # correctly escaped non-ASCII characters (\X2\00ED\X0\) are valid STEP and must pass + SyntaxValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='pass_non_ascii_x2_escape_header.ifc', + file='pass_non_ascii_x2_escape_header.ifc', + size=os.path.getsize('apps/ifc_validation/fixtures/pass_non_ascii_x2_escape_header.ifc'), + ) + request.mark_as_initiated() + + header_syntax_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name, + ) + + outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id) + self.assertEqual(len(outcomes), 1) + self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.PASSED) + self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.PASSED) + def test_determine_aggregate_status_for_multiple_outcomes(self): # test cases diff --git a/backend/apps/ifc_validation/tests/tests_header_validation_task.py b/backend/apps/ifc_validation/tests/tests_header_validation_task.py index 0b3af4e6..f48c15bf 100644 --- a/backend/apps/ifc_validation/tests/tests_header_validation_task.py +++ b/backend/apps/ifc_validation/tests/tests_header_validation_task.py @@ -207,3 +207,30 @@ def test_header_validation_task_correctly_parses_existing_authoring_tool2(self): self.assertEquals('MyFabTool', model.produced_by.name) self.assertEquals('2025.1', model.produced_by.version) self.assertEquals('Acme Inc.', model.produced_by.company.name) + + def test_header_validation_task_decodes_step_escapes(self): + + # arrange + HeaderValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='pass_non_ascii_x2_escape_header.ifc', + file='pass_non_ascii_x2_escape_header.ifc', + size=1 + ) + request.mark_as_initiated() + + # act + header_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name + ) + + # assert + model = Model.objects.get(id=request.id) + self.assertIsNotNone(model) + self.assertEqual(model.status_header, Model.Status.VALID) + self.assertEqual('Buhodra Ingeniería S.A.', model.header_validation.get('company_name')) + self.assertEqual('Buhodra Ingeniería S.A. - Istram - 26.06', model.header_validation.get('originating_system')) + self.assertNotIn('\\X2\\', str(model.header_validation)) + self.assertEquals('Buhodra Ingeniería S.A.', model.produced_by.company.name) diff --git a/backend/apps/ifc_validation/tests/tests_syntax_validation_task.py b/backend/apps/ifc_validation/tests/tests_syntax_validation_task.py index eb569c68..3a430b03 100644 --- a/backend/apps/ifc_validation/tests/tests_syntax_validation_task.py +++ b/backend/apps/ifc_validation/tests/tests_syntax_validation_task.py @@ -81,6 +81,58 @@ def test_syntax_validation_task_creates_error_for_utf8_bom(self): self.assertIn("character", outcomes.first().observed.lower()) + def test_syntax_validation_task_translates_unicode_decode_error(self): + + # latin-1 encoded í crashes the parser; the raw traceback must never reach the user + SyntaxValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='fail_non_ascii_latin1_header.ifc', + file='fail_non_ascii_latin1_header.ifc', + size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_latin1_header.ifc'), + ) + request.mark_as_initiated() + + syntax_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name, + ) + + outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id) + self.assertEqual(len(outcomes), 1) + self.assertEqual(outcomes.first().severity, ValidationOutcome.OutcomeSeverity.ERROR) + self.assertEqual(outcomes.first().outcome_code, ValidationOutcome.ValidationOutcomeCode.SYNTAX_ERROR) + observed = outcomes.first().observed + self.assertNotIn('UnicodeDecodeError', observed) + self.assertNotIn('Traceback', observed) + self.assertIn('On line 4 column', observed) + self.assertIn("non-ASCII byte ('0xed')", observed) + self.assertIn('\\X2\\', observed) + + def test_syntax_validation_task_reports_correct_position_for_raw_utf8(self): + + SyntaxValidationTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='fail_non_ascii_raw_utf8_header.ifc', + file='fail_non_ascii_raw_utf8_header.ifc', + size=os.path.getsize('apps/ifc_validation/fixtures/fail_non_ascii_raw_utf8_header.ifc'), + ) + request.mark_as_initiated() + + syntax_validation_subtask( + prev_result={'is_valid': True, 'reason': 'test'}, + id=request.id, + file_name=request.file_name, + ) + + outcomes = ValidationOutcome.objects.filter(validation_task__request_id=request.id) + self.assertEqual(len(outcomes), 1) + observed = outcomes.first().observed + self.assertIn("Unexpected character ('0xed')", observed) + # the í is in FILE_NAME on line 4; the reported line must point there + self.assertIn('On line 4 column', observed) + self.assertNotIn('UnicodeDecodeError', observed) + def test_determine_aggregate_status_for_multiple_outcomes(self): # test cases